A little cleanup

diff --git a/dump.sh b/dump.sh
index 2bf9710f71451bea92e1d5bec94af5f32cd15d0f..4532e4729447b259f4f3b3dda5124838b43fe6d6 100755 (executable)
--- a/dump.sh
+++ b/dump.sh
@@ -1,9 +1,9 @@
-#!/bin/sh
+#!/bin/bash

 
 

-FROM=54696
+FROM=1

 #FROM=190000
 
 #FROM=190000
 

-UNTIL=55060
+UNTIL=1000000

 #UNTIL=200000
 
 cat <<EOF
 #UNTIL=200000
 
 cat <<EOF


@@ -11,18 +11,21 @@ cat <<EOF
 2. Visit https://dvr.dsk.gv.at/at.gv.bka.dvr.public/DVRRecherche.aspx and
    search for "Bundeskanzleramt" in "Auftraggeber". Then press
    "Datenanwendung". This is important in order to fetch the purposes too!
 2. Visit https://dvr.dsk.gv.at/at.gv.bka.dvr.public/DVRRecherche.aspx and
    search for "Bundeskanzleramt" in "Auftraggeber". Then press
    "Datenanwendung". This is important in order to fetch the purposes too!

-3. Loof for a cookie from "dvr.dsk.gv.at" named "ASP.NET_SessionId". Copy its
+3. Look for a cookie from "dvr.dsk.gv.at" named "ASP.NET_SessionId". Copy its

    content in this script at the curresponding curl line.
 EOF
 
    content in this script at the curresponding curl line.
 EOF
 

-for id in `seq ${FROM} ${UNTIL}`; do
-    
+COOKIE_SESSION_ID=0ckafib0jrhs3ayhd3m55bv0
+
+pause
+
+for id in $(seq ${FROM} ${UNTIL}); do

     echo -n "### ID ${id}"
 
     FN="dump/$id.html.gz"
     if ! [ -e "${FN}" ]; then
         echo -n " Fetching..."
     echo -n "### ID ${id}"
 
     FN="dump/$id.html.gz"
     if ! [ -e "${FN}" ]; then
         echo -n " Fetching..."

-        curl -s -b ASP.NET_SessionId=0ckafib0jrhs3ayhd3m55bv0 \
+        curl -s -b ASP.NET_SessionId=${COOKIE_SESSION_ID} \

             "https://dvr.dsk.gv.at/at.gv.bka.dvr.public/AuftraggeberDetail.aspx?Id=${id}" | gzip > "${FN}"
 
         # Do not stress server too much
             "https://dvr.dsk.gv.at/at.gv.bka.dvr.public/AuftraggeberDetail.aspx?Id=${id}" | gzip > "${FN}"
 
         # Do not stress server too much

diff --git a/dvr-managedb b/dvr-managedb
index aaf68e22151e73d03e6fc7d96c4ae1045f2b3333..fbe9766080a395c1b914bf98d8238650a3bd93f1 100755 (executable)
--- a/dvr-managedb
+++ b/dvr-managedb
@@ -76,9 +76,9 @@ class GeolocationProviders(object):
     """Manages a list of geolocation providers."""
 
     providers = [
     """Manages a list of geolocation providers."""
 
     providers = [

-            GoogleGeolocation(),
-            BingGeolocation()
-            ]
+        GoogleGeolocation(),
+        BingGeolocation()
+    ]

 
     @staticmethod
     def isNameValid(name):
 
     @staticmethod
     def isNameValid(name):


@@ -93,15 +93,16 @@ class GeolocationProviders(object):
     @staticmethod
     def getByIndex(idx):
         """Get provider by index"""
     @staticmethod
     def getByIndex(idx):
         """Get provider by index"""

-        assert(GeolocationProviders.isIndexValid(idx))
+        assert (GeolocationProviders.isIndexValid(idx))

         return GeolocationProviders.providers[idx]
 
     @staticmethod
     def getIndexByName(name):
         """Get provider by its name"""
         return GeolocationProviders.providers[idx]
 
     @staticmethod
     def getIndexByName(name):
         """Get provider by its name"""

-        assert(GeolocationProviders.isNameValid(name))
+        assert (GeolocationProviders.isNameValid(name))

         return [p.name().lower() for p in GeolocationProviders.providers].index(name.lower())
 
         return [p.name().lower() for p in GeolocationProviders.providers].index(name.lower())
 

+

 class Database:
     """Encapsualtes a DVR database."""
 
 class Database:
     """Encapsualtes a DVR database."""
 


@@ -239,10 +240,10 @@ class Database:
 
     def get_geolocation(self, dvrid, provider):
         """Return geolocation of given DVR-ID and provider."""
 
     def get_geolocation(self, dvrid, provider):
         """Return geolocation of given DVR-ID and provider."""

-        assert(GeolocationProviders.isIndexValid(provider))
+        assert (GeolocationProviders.isIndexValid(provider))

 
         c = self.conn.cursor()
 
         c = self.conn.cursor()

-        c.execute("SELECT * FROM geolocations WHERE dvrid=? AND provider=?", \
+        c.execute("SELECT * FROM geolocations WHERE dvrid=? AND provider=?",

                   (dvrid, provider))
         res = c.fetchone()
         if res is None:
                   (dvrid, provider))
         res = c.fetchone()
         if res is None:


@@ -251,10 +252,10 @@ class Database:
 
     def add_geolocation(self, dvrid, provider, lat, lon):
         """Add a geolocatoin for a given DVR-ID."""
 
     def add_geolocation(self, dvrid, provider, lat, lon):
         """Add a geolocatoin for a given DVR-ID."""

-        assert(GeolocationProviders.isIndexValid(provider))
+        assert (GeolocationProviders.isIndexValid(provider))

 
         c = self.conn.cursor()
 
         c = self.conn.cursor()

-        c.execute("INSERT INTO geolocations VALUES (?, ?, ?, ?)", \
+        c.execute("INSERT INTO geolocations VALUES (?, ?, ?, ?)",

                   (dvrid, provider, lat, lon))
         c.close()
         self.conn.commit()
                   (dvrid, provider, lat, lon))
         c.close()
         self.conn.commit()


@@ -436,7 +437,7 @@ def processAdd(db, source):
             for (dirpath, dirnames, filenames) in os.walk(source):
                 for f in filenames:
                     if f.endswith(".html") or f.endswith(".html.gz"):
             for (dirpath, dirnames, filenames) in os.walk(source):
                 for f in filenames:
                     if f.endswith(".html") or f.endswith(".html.gz"):

-                        p = os.path.join(dirpath,f)
+                        p = os.path.join(dirpath, f)

                         processAdd(db, p)
         else:
             print("What the hell is '%s'?" % source, file=sys.stderr)
                         processAdd(db, p)
         else:
             print("What the hell is '%s'?" % source, file=sys.stderr)


@@ -561,7 +562,7 @@ if __name__ == "__main__":
                 maptemplatefn = arg
             else:
                 print("Unknown option '", opt, "'.", file=sys.stderr)
                 maptemplatefn = arg
             else:
                 print("Unknown option '", opt, "'.", file=sys.stderr)

-                assert(False)
+                assert (False)

 
     except getopt.GetoptError as e:
         print("Error parsing arguments:", e, file=sys.stderr)
 
     except getopt.GetoptError as e:
         print("Error parsing arguments:", e, file=sys.stderr)