X-Git-Url: http://gitweb.michael.orlitzky.com/?p=dead%2Fcensus-tools.git;a=blobdiff_plain;f=makefile;h=74729735303716524f63a12029e89bf658c6064a;hp=f3136a6cda47c33afcb4d13b3d9a8cde272ae5ff;hb=d797c850a8493c0027b1e8b8dbc4d3e78840d4d9;hpb=84d512e317cac78d67bec5c64fcf761d3f5e0689 diff --git a/makefile b/makefile index f3136a6..7472973 100644 --- a/makefile +++ b/makefile @@ -1,9 +1,191 @@ -.PHONY : test +DB_NAME=census +DB_USER=postgres +TIGER_SRID=4269 +SHAPELY_URL=http://pypi.python.org/packages/source/S/Shapely/Shapely-1.0.14.tar.gz + + +# Dark magic. We set these makefile variables to be the result of the +# 'shell' function. The shell function, in turn, executes a Python +# script which determines the locations of these files. +SHP2PGSQL := $(shell bin/find_file_paths --root /usr --single shp2pgsql) +POSTGIS_SQL := $(shell bin/find_file_paths --root /usr lwpostgis.sql postgis.sql) +SPATIAL_REF_SYS_SQL := $(shell bin/find_file_paths --root /usr spatial_ref_sys.sql) + +# Necessary to run test/data without prerequisites. +# +.PHONY : test data lib + + +# The default task, since it comes first in the list. +# +all: clean lib test -all: clean test test: ./bin/run_tests + +# Download or check out any third-party libraries. +lib: + if [ ! -d lib/Shapely ]; then \ + wget -O shapely.tar.gz $(SHAPELY_URL); \ + tar -xvzf shapely.tar.gz -C lib/ ; \ + rm shapely.tar.gz; \ + mv lib/Shapely* lib/Shapely; \ + fi; + + +# Remove byte-compiled python code. +# clean: find ./ -name '*.pyc' -print0 | xargs -0 rm -f + + +data: + bin/download_data + + +# There is a small issue here with the blocks_db and lines_db +# targets. Each of these requires that the database exists, and might +# therefore depend on the newdb target. However, if /each/ of them +# depends on newdb, the database will be dropped twice and the data +# from one of {blocks, lines} would be lost. +# +# We therefore assume that the database already exists when blocks_db +# or lines_db are initiated. +blocks_db: data blocks_table +# All Blocks +# +# The table already exists, so we can append to it, and we don't have +# to create the GiST index. + for state in data/census2000/*; do \ + $(SHP2PGSQL) \ + -a \ + -s $(TIGER_SRID) \ + -D \ + $$state/blocks/*.shp \ + tiger_blocks \ + | psql -U $(DB_USER) -d $(DB_NAME); \ + done; + +# Summary File 1 +# +# Run all of the geo (uf1) files through the import script. This has +# to happen after the blocks import since we impose a foreign key +# restriction. + for state in data/census2000/*; do \ + bin/sf1blocks2sql $$state/sf1/*.uf1 sf1_blocks \ + | psql -U $(DB_USER) -d $(DB_NAME) \ + > /dev/null; \ + done; + +# Run the query to combine the two blocks tables, and drop the +# constituents. + psql -U $(DB_USER) \ + -d $(DB_NAME) \ + -f sql/combine-block-tables.sql + + +lines_db: data tiger_lines_table +# All Lines +# +# Since the table and index already exist, we can utilize -a, +# and leave -I out. + for state in data/census2000/*; do \ + for shapefile in $$state/lines/*.shp; do \ + echo "Importing $$shapefile."; \ + $(SHP2PGSQL) \ + -a \ + -s $(TIGER_SRID) \ + $$shapefile \ + tiger_lines \ + | bin/filter-transactions \ + | psql -U $(DB_USER) -d $(DB_NAME) \ + > /dev/null; \ + done; \ + done; + + + +# This imports the Tiger data using shp2pgsql. The shapefiles +# should exist, since this task depends on the "data" task, which +# downloads said shapefiles. +# +# After the TIGER import is done, we use the sf1blocks2sql script to +# parse and import the geographic header record information. +# +db: newdb blocks_data lines_data + # Do nothing except fulfill our prerequisites. + + + +# First, we drop and re-create the DB_NAME database (or schema, +# whatever). Then, we add PL/pgSQL support to the database. +# +# At that point, we import the two PostGIS files, postgis.sql and +# spatial_ref_sys.sql. The postgis.sql file contains the geometry +# functions, while spatial_ref_sys.sql contains a table of SRIDs, and +# their associated properties. PostGIS requires both. +# +newdb: +# Ignore the result of dropdb when it fails. + dropdb -U $(DB_USER) $(DB_NAME) || true + createdb -U $(DB_USER) $(DB_NAME) + createlang -U $(DB_USER) plpgsql $(DB_NAME) + + psql -d $(DB_NAME) \ + -U $(DB_USER) \ + -f $(POSTGIS_SQL) \ + > /dev/null + + psql -d $(DB_NAME) \ + -U $(DB_USER) \ + -f $(SPATIAL_REF_SYS_SQL) \ + > /dev/null + + +# This just runs the SQL script to create the sf1_blocks table. +sf1_blocks_table: + psql -d $(DB_NAME) \ + -U $(DB_USER) \ + -f sql/create-sf1_blocks-table.sql \ + > /dev/null + + +# Create the tiger_blocks table, and create its GiST index. Having the +# table already exist makes importing via shp2pgsql much easier. +# Any blocks file will work as an argument. +tiger_blocks_table: + $(SHP2PGSQL) \ + -p \ + -I \ + -s $(TIGER_SRID) \ + data/census2000/maryland/blocks/tl_2009_24_tabblock00.shp \ + tiger_blocks \ + | psql -U $(DB_USER) -d $(DB_NAME) \ + > /dev/null + +# Create the "blocks" table, which is the result of joining +# the tiger_blocks and sf1_blocks tables. +blocks_table: tiger_blocks_table sf1_blocks_table + psql -U $(DB_USER) \ + -d $(DB_NAME) \ + -f sql/create-blocks-table.sql + + +# Prepare the tiger_lines table, and create the GiST index on its +# geometry column. Any lines shapefile will do here. +tiger_lines_table: + $(SHP2PGSQL) \ + -p \ + -I \ + -s $(TIGER_SRID) \ + data/census2000/maryland/lines/tl_2009_24510_edges.shp \ + tiger_lines \ + | psql -U $(DB_USER) -d $(DB_NAME) \ + > /dev/null + +# Add a unique index on the "tlid" column. + psql -U $(DB_USER) \ + -d $(DB_NAME) \ + -f sql/create_tlid_unique_index.sql