]> gitweb.michael.orlitzky.com - dead/census-tools.git/commitdiff
Added the MD lines import to the makefile.
authorMichael Orlitzky <michael@orlitzky.com>
Sun, 4 Oct 2009 17:18:51 +0000 (13:18 -0400)
committerMichael Orlitzky <michael@orlitzky.com>
Sun, 4 Oct 2009 17:18:51 +0000 (13:18 -0400)
Reworked the data target to be more modular.
Made the blocks/lines import via wildcard.
Fixed all of the index/append inconsistencies in the shp2pgsql usage.

makefile

index 2ab9dbae7b627e27f927061b3dda062abcfd43b0..d32c27f81ef2467a6b513e4fbb4876b494c3a101 100644 (file)
--- a/makefile
+++ b/makefile
@@ -133,7 +133,7 @@ ny_blocks:
        fi;
 
 
-tiger_lines: dc_lines
+tiger_lines: dc_lines md_lines
 
 dc_lines:
        mkdir -p data/census2000/dc/lines
@@ -144,95 +144,322 @@ dc_lines:
                rm dclines.zip;                                         \
        fi;
 
-# This task does a couple of things. First, it drops and re-creates
-# the DB_NAME database (or schema, whatever). Then, it adds PL/pgSQL
-# support to the database.
-#
-# At that point, we import the two PostGIS files, postgis.sql and
-# spatial_ref_sys.sql. The postgis.sql file contains the geometry
-# functions, while spatial_ref_sys.sql contains a table of SRIDs, and
-# their associated properties. PostGIS requires both.
-#
-# Then, we import the Tiger data using shp2pgsql. The shapefiles
-# should exist, since this task depends on the "data" task, which
-# downloads said shapefiles.
-#
-# Finally, we create the table for the demographic data (obtained from
-# the geographic header records), and populate that table with the output
-# of the sf1blocks2sql script.
-#
-db: data
-       # Ignore the result of dropdb when it fails.
-       dropdb -U $(DB_USER) $(DB_NAME) || true
-       createdb -U $(DB_USER) $(DB_NAME)
-       createlang -U $(DB_USER) plpgsql $(DB_NAME)
+md_lines: allegany_lines anne_arundel_lines baltimore_co_lines baltimore_ci_lines calvert_lines caroline_lines carroll_lines cecil_lines charles_lines dorchester_lines frederick_lines garrett_lines harford_lines howard_lines kent_lines montgomery_lines prince_georges_lines queen_annes_lines st_marys_lines somerset_lines talbot_lines washington_lines wicomico_lines worcester_lines
 
-       psql -d $(DB_NAME) \
-             -U $(DB_USER) \
-             -f $(PG_SHAREDIR)/contrib/postgis.sql
+allegany_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24001_edges.shp ];  \
+       then                                                                 \
+               wget -O alleganylines.zip $(ALLEGANY_LINES_URL);             \
+               unzip alleganylines.zip -d ./data/census2000/maryland/lines; \
+               rm alleganylines.zip;                                        \
+       fi;
+
+anne_arundel_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24003_edges.shp ];  \
+       then                                                                 \
+               wget -O aalines.zip $(ANNE_ARUNDEL_LINES_URL);               \
+               unzip aalines.zip -d ./data/census2000/maryland/lines;       \
+               rm aalines.zip;                                              \
+       fi;
 
-       psql -d $(DB_NAME) \
-             -U $(DB_USER) \
-             -f $(PG_SHAREDIR)/contrib/spatial_ref_sys.sql
+baltimore_co_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24005_edges.shp ];  \
+       then                                                                 \
+               wget -O bcolines.zip $(BALTIMORE_CO_LINES_URL);              \
+               unzip bcolines.zip -d ./data/census2000/maryland/lines;      \
+               rm bcolines.zip;                                             \
+       fi;
+
+baltimore_ci_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24510_edges.shp ];  \
+       then                                                                 \
+               wget -O bcilines.zip $(BALTIMORE_CI_LINES_URL);              \
+               unzip bcilines.zip -d ./data/census2000/maryland/lines;      \
+               rm bcilines.zip;                                             \
+       fi;
+
+calvert_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24009_edges.shp ];  \
+       then                                                                 \
+               wget -O calvertlines.zip $(CALVERT_LINES_URL);               \
+               unzip calvertlines.zip -d ./data/census2000/maryland/lines;  \
+               rm calvertlines.zip;                                         \
+       fi;
+
+caroline_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24011_edges.shp ];  \
+       then                                                                 \
+               wget -O carolinelines.zip $(CAROLINE_LINES_URL);             \
+               unzip carolinelines.zip -d ./data/census2000/maryland/lines; \
+               rm carolinelines.zip;                                        \
+       fi;
+
+carroll_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24013_edges.shp ];  \
+       then                                                                 \
+               wget -O carrolllines.zip $(CARROLL_LINES_URL);               \
+               unzip carrolllines.zip -d ./data/census2000/maryland/lines;  \
+               rm carrolllines.zip;                                         \
+       fi;
+
+cecil_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24015_edges.shp ];  \
+       then                                                                 \
+               wget -O cecillines.zip $(CECIL_LINES_URL);                   \
+               unzip cecillines.zip -d ./data/census2000/maryland/lines;    \
+               rm cecillines.zip;                                           \
+       fi;
+
+charles_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24017_edges.shp ];  \
+       then                                                                 \
+               wget -O charleslines.zip $(CHARLES_LINES_URL);               \
+               unzip charleslines.zip -d ./data/census2000/maryland/lines;  \
+               rm charleslines.zip;                                         \
+       fi;
+
+dorchester_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24019_edges.shp ];    \
+       then                                                                   \
+               wget -O dorchesterlines.zip $(DORCHESTER_LINES_URL);           \
+               unzip dorchesterlines.zip -d ./data/census2000/maryland/lines; \
+               rm dorchesterlines.zip;                                        \
+       fi;
+
+frederick_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24021_edges.shp ];   \
+       then                                                                  \
+               wget -O fredericklines.zip $(FREDERICK_LINES_URL);            \
+               unzip fredericklines.zip -d ./data/census2000/maryland/lines; \
+               rm fredericklines.zip;                                        \
+       fi;
 
-# D.C. Blocks
+garrett_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24023_edges.shp ]; \
+       then                                                                \
+               wget -O garrettlines.zip $(GARRETT_LINES_URL);              \
+               unzip garrettlines.zip -d ./data/census2000/maryland/lines; \
+               rm garrettlines.zip;                                        \
+       fi;
 
-       $(PG_BINDIR)/shp2pgsql                                        \
-                 -I                                                  \
-                 -s $(TIGER_SRID)                                    \
-                 -D                                                  \
-                 data/census2000/dc/block/tl_2009_11_tabblock00.shp  \
-                 tiger_blocks                                        \
-                 | psql -U $(DB_USER) -d $(DB_NAME)
+harford_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24025_edges.shp ]; \
+       then                                                                \
+               wget -O harfordlines.zip $(HARFORD_LINES_URL);              \
+               unzip harfordlines.zip -d ./data/census2000/maryland/lines; \
+               rm harfordlines.zip;                                        \
+       fi;
 
-# Maryland Blocks
+howard_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24027_edges.shp ]; \
+       then                                                                \
+               wget -O howardlines.zip $(HOWARD_LINES_URL);                \
+               unzip howardlines.zip -d ./data/census2000/maryland/lines;  \
+               rm howardlines.zip;                                         \
+       fi;
 
-       $(PG_BINDIR)/shp2pgsql                                              \
-                 -I                                                        \
-                 -s $(TIGER_SRID)                                          \
-                 -D                                                        \
-                 data/census2000/maryland/block/tl_2009_24_tabblock00.shp  \
-                 tiger_blocks                                              \
-                 | psql -U $(DB_USER) -d $(DB_NAME)
+kent_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24029_edges.shp ]; \
+       then                                                                \
+               wget -O kentlines.zip $(KENT_LINES_URL);                    \
+               unzip kentlines.zip -d ./data/census2000/maryland/lines;    \
+               rm kentlines.zip;                                           \
+       fi;
 
+montgomery_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24031_edges.shp ];    \
+       then                                                                   \
+               wget -O montgomerylines.zip $(MONTGOMERY_LINES_URL);           \
+               unzip montgomerylines.zip -d ./data/census2000/maryland/lines; \
+               rm montgomerylines.zip;                                        \
+       fi;
 
-# Virginia Blocks
+prince_georges_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24033_edges.shp ]; \
+       then                                                                \
+               wget -O pglines.zip $(PRINCE_GEORGES_LINES_URL);            \
+               unzip pglines.zip -d ./data/census2000/maryland/lines;      \
+               rm pglines.zip;                                             \
+       fi;
 
-       $(PG_BINDIR)/shp2pgsql -a                                           \
-                 -I                                                        \
-                 -s $(TIGER_SRID)                                          \
-                 -D                                                        \
-                 data/census2000/virginia/block/tl_2009_51_tabblock00.shp  \
-                 tiger_blocks                                              \
-                 | psql -U $(DB_USER) -d $(DB_NAME)
+queen_annes_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24035_edges.shp ]; \
+       then                                                                \
+               wget -O qalines.zip $(QUEEN_ANNES_LINES_URL);               \
+               unzip qalines.zip -d ./data/census2000/maryland/lines;      \
+               rm qalines.zip;                                             \
+       fi;
 
+st_marys_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24037_edges.shp ]; \
+       then                                                                \
+               wget -O smlines.zip $(ST_MARYS_LINES_URL);                  \
+               unzip smlines.zip -d ./data/census2000/maryland/lines;      \
+               rm smlines.zip;                                             \
+       fi;
 
-# Pennsylvania Blocks
+somerset_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24039_edges.shp ];  \
+       then                                                                 \
+               wget -O somersetlines.zip $(SOMERSET_LINES_URL);             \
+               unzip somersetlines.zip -d ./data/census2000/maryland/lines; \
+               rm somersetlines.zip;                                        \
+       fi;
 
-       $(PG_BINDIR)/shp2pgsql -a                                               \
-                 -I                                                            \
-                 -s $(TIGER_SRID)                                              \
-                 -D                                                            \
-                 data/census2000/pennsylvania/block/tl_2009_42_tabblock00.shp  \
-                 tiger_blocks                                                  \
-                  | psql -U $(DB_USER) -d $(DB_NAME)
+talbot_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24041_edges.shp ]; \
+       then                                                                \
+               wget -O talbotlines.zip $(TALBOT_LINES_URL);                \
+               unzip talbotlines.zip -d ./data/census2000/maryland/lines;  \
+               rm talbotlines.zip;                                         \
+       fi;
 
+washington_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24043_edges.shp ];    \
+       then                                                                   \
+               wget -O washingtonlines.zip $(WASHINGTON_LINES_URL);           \
+               unzip washingtonlines.zip -d ./data/census2000/maryland/lines; \
+               rm washingtonlines.zip;                                        \
+       fi;
 
-# New York Blocks
+wicomico_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24045_edges.shp ];  \
+       then                                                                 \
+               wget -O wicomicolines.zip $(WICOMICO_LINES_URL);             \
+               unzip wicomicolines.zip -d ./data/census2000/maryland/lines; \
+               rm wicomicolines.zip;                                        \
+       fi;
 
-       $(PG_BINDIR)/shp2pgsql -a                                           \
-                 -I                                                        \
-                 -s $(TIGER_SRID)                                          \
-                 -D                                                        \
-                 data/census2000/new_york/block/tl_2009_36_tabblock00.shp  \
-                 tiger_blocks                                              \
-                  | psql -U $(DB_USER) -d $(DB_NAME)
+worcester_lines:
+       mkdir -p data/census2000/maryland/lines
+       if [ ! -f data/census2000/maryland/lines/tl_2009_24047_edges.shp ];   \
+       then                                                                  \
+               wget -O worcesterlines.zip $(WORCESTER_LINES_URL);            \
+               unzip worcesterlines.zip -d ./data/census2000/maryland/lines; \
+               rm worcesterlines.zip;                                        \
+       fi;
 
 
-       psql -d $(DB_NAME) \
-             -U $(DB_USER) \
-             -f sql/create-sf1_blocks-table.sql
+# This imports the Tiger data using shp2pgsql. The shapefiles
+# should exist, since this task depends on the "data" task, which
+# downloads said shapefiles.
+#
+# After the TIGER import is done, we use the sf1blocks2sql script to
+# parse and import the geographic header record information.
+#
+db: data newdb tiger_blocks_table tiger_lines_table sf1_blocks_table
+# All Blocks
+#
+# The table already exists, so we can append to it, and we don't have
+# to create the GiST index.
+       for state in data/census2000/*; do                  \
+               $(PG_BINDIR)/shp2pgsql                      \
+                       -a                                  \
+                       -s $(TIGER_SRID)                    \
+                       -D                                  \
+                       $$state/block/*.shp                 \
+                       tiger_blocks                        \
+                       | psql -U $(DB_USER) -d $(DB_NAME); \
+       done;
+
+# MD Lines
+#
+# Since the table and index already exist, we can utilize -a,
+# and leave -I out.
+       for x in data/census2000/maryland/lines/*.shp; do   \
+               $(PG_BINDIR)/shp2pgsql                      \
+                       -a                                  \
+                       -s $(TIGER_SRID)                    \
+                       -D                                  \
+                       $$x                                 \
+                       tiger_lines                         \
+                       | psql -U $(DB_USER) -d $(DB_NAME); \
+       done;
 
        bin/sf1blocks2sql src/Tests/Fixtures/SummaryFile1/mdgeo.uf1 sf1_blocks \
-                          | psql -U postgres -d $(DB_NAME) > /dev/null
+                          | psql -U postgres -d $(DB_NAME)                     \
+                          > /dev/null
+
+
+
+# First, we drop and re-create the DB_NAME database (or schema,
+# whatever). Then, we add PL/pgSQL support to the database.
+#
+# At that point, we import the two PostGIS files, postgis.sql and
+# spatial_ref_sys.sql. The postgis.sql file contains the geometry
+# functions, while spatial_ref_sys.sql contains a table of SRIDs, and
+# their associated properties. PostGIS requires both.
+#
+newdb:
+# Ignore the result of dropdb when it fails.
+       dropdb -U $(DB_USER) $(DB_NAME) || true
+       createdb -U $(DB_USER) $(DB_NAME)
+       createlang -U $(DB_USER) plpgsql $(DB_NAME)
 
+       psql -d $(DB_NAME)                         \
+             -U $(DB_USER)                         \
+             -f $(PG_SHAREDIR)/contrib/postgis.sql \
+             > /dev/null
+
+       psql -d $(DB_NAME)                                 \
+             -U $(DB_USER)                                 \
+             -f $(PG_SHAREDIR)/contrib/spatial_ref_sys.sql \
+            > /dev/null
+
+
+# This just runs the SQL script to create the sf1_blocks table.
+sf1_blocks_table:
+       psql -d $(DB_NAME)                      \
+             -U $(DB_USER)                      \
+             -f sql/create-sf1_blocks-table.sql \
+            > /dev/null
+
+
+# Create the tiger_blocks table, and create its GiST index. Having the
+# table already exist makes importing via shp2pgsql much easier.
+# Any blocks file will work as an argument.
+tiger_blocks_table:
+       $(PG_BINDIR)/shp2pgsql                                            \
+               -p                                                        \
+               -I                                                        \
+               -s $(TIGER_SRID)                                          \
+               data/census2000/maryland/block/tl_2009_24_tabblock00.shp  \
+               tiger_blocks                                              \
+               | psql -U postgres -d $(DB_NAME)                          \
+                > /dev/null
+
+
+# Prepare the tiger_lines table, and create the GiST index on its
+# geometry column. Any lines shapefile will do here.
+tiger_lines_table:
+       $(PG_BINDIR)/shp2pgsql                                         \
+               -p                                                     \
+               -I                                                     \
+               -s $(TIGER_SRID)                                       \
+               data/census2000/maryland/lines/tl_2009_24510_edges.shp \
+               tiger_lines                                            \
+               | psql -U postgres -d $(DB_NAME)                       \
+                > /dev/null