convert_osm_to_sqlite.sh

   1 #! /bin/bash
   2
   3 set -o nounset
   4 set -o errexit
   5
   6 ROOT=$(dirname $0)
   7
   8 NEW=1
   9 DUPE=0
  10
  11 while getopts "rDo:d:" FLAG ; do
  12     case "$FLAG" in
  13         r) NEW=0 ;;
  14         d) DATABASE_FILE=$OPTARG ;;
  15         o) OSM_FILE=$OPTARG ;;
  16         D) DUPE=1 ;;
  17     esac
  18
  19 done
  20
  21 if [[ $NEW = 1 ]] ; then
  22     rm -f ${DATABASE_FILE}
  23     sqlite3 ${DATABASE_FILE} < ${ROOT}/create_database.sql
  24 fi
  25
  26 # TODO check if file is bzipped and unzip it
  27
  28 # TODO check if pv is not installed and use cat instead
  29
  30 pv -N "extracting from osm" ${OSM_FILE} | python ${ROOT}/osm2tsvs.py
  31
  32 if [[ $DUPE = 1 ]] ; then
  33     sqlite3 ${DATABASE_FILE} < ${ROOT}/create_tmp_database.sql
  34 fi
  35
  36 function import_file() {
  37     TABLE=$1
  38     rm -f fifo
  39     mkfifo fifo
  40     ( pv -N "importing $TABLE" ${TABLE}.tsv.gz | zcat > fifo ) &
  41     if [[ $DUPE = 0 ]] ; then
  42         if [[ $NEW = 1 ]] ; then
  43             echo -e ".separator \"\\\t\"\n.import fifo $TABLE" | sqlite3 ${DATABASE_FILE}
  44         elif [[ $NEW = 0 ]] ; then
  45             echo -e "create table tmp_${TABLE} as select * from ${TABLE} limit 0;\n.separator \"\\\t\"\n.import fifo tmp_$TABLE\ninsert or replace into ${TABLE} select * from tmp_${TABLE};" | sqlite3 ${DATABASE_FILE}
  46         fi
  47     elif [[ $DUPE = 1 ]] ; then
  48         echo -e ".separator \"\\\t\"\n.import fifo tmp_$TABLE" | sqlite3 ${DATABASE_FILE}
  49         rm -f fifo
  50         echo "removing duplicates from $TABLE..."
  51         sqlite3 ${DATABASE_FILE} "insert or ignore into ${TABLE} select * from tmp_${TABLE};"
  52     fi
  53
  54     rm -f fifo
  55 }
  56
  57 import_file nodes
  58 import_file node_tags
  59
  60 import_file ways
  61 import_file way_tags
  62 import_file way_nodes
  63
  64 import_file relations
  65 import_file relation_tags
  66 import_file relation_members
  67
  68 rm -f nodes.tsv.gz node_tags.tsv.gz ways.tsv.gz way_tags.tsv.gz way_nodes.tsv.gz relations.tsv.gz relation_tags.tsv.gz relation_members.tsv.gz fifo
  69
  70 if [[ $DUPE = 1 ]] ; then
  71     sqlite3 ${DATABASE_FILE} "drop table tmp_nodes;"
  72     sqlite3 ${DATABASE_FILE} "drop table tmp_node_tags;"
  73     sqlite3 ${DATABASE_FILE} "drop table tmp_ways;"
  74     sqlite3 ${DATABASE_FILE} "drop table tmp_way_tags;"
  75     sqlite3 ${DATABASE_FILE} "drop table tmp_way_nodes;"
  76     sqlite3 ${DATABASE_FILE} "drop table tmp_relations;"
  77     sqlite3 ${DATABASE_FILE} "drop table tmp_relation_tags;"
  78     sqlite3 ${DATABASE_FILE} "drop table tmp_relation_members;"
  79 fi
  80
  81 sqlite3 ${DATABASE_FILE} "VACUUM;"