trackdirect-hmk/server/trackdirect/database/PacketTableCreator.py

import logging
from twisted.python import log
import psycopg2
import psycopg2.extras
import datetime
import time
import calendar

from trackdirect.database.DatabaseObjectFinder import DatabaseObjectFinder
from trackdirect.exceptions.TrackDirectMissingTableError import TrackDirectMissingTableError


class PacketTableCreator():
    """The PacketTableCreator class handles packet table name logic

    Note:
        Packets are stored in different tables depending on what day they are received,
        new packet tables are created by this class.
    """

    def __init__(self, db):
        """The __init__ method.

        Args:
            db (psycopg2.Connection): Database connection
        """
        self.db = db
        self.dbObjectFinder = DatabaseObjectFinder(db)
        self.logger = logging.getLogger('trackdirect')
        self.createIfMissing = True

    def disableCreateIfMissing(self):
        """Disable feature that creates new tables if missing
        """
        self.createIfMissing = False

    def enableCreateIfMissing(self):
        """Enable feature that creates new tables if missing
        """
        self.createIfMissing = True

    def getPacketTable(self, packetTimestamp):
        """Returns the name of the packet table

        Args:
            packetTimestamp (int): Unix timestamp that we need the table for

        Returns:
            Returns the name of the packet table as a string
        """
        date = datetime.datetime.utcfromtimestamp(
            packetTimestamp).strftime('%Y%m%d')
        packetTable = 'packet' + date
        if (not self.dbObjectFinder.checkTableExists(packetTable)):
            if(self.createIfMissing):
                minTimestamp = packetTimestamp // (24*60*60) * (24*60*60)
                maxTimestamp = minTimestamp + (24*60*60)
                self._createPacketTable(
                    packetTable, minTimestamp, maxTimestamp)
                self.dbObjectFinder.setTableExists(packetTable)
            else:
                raise TrackDirectMissingTableError(
                    'Database table ' + packetTable + ' does not exists')
        return packetTable

    def getPacketTables(self, startTimestamp, endTimestamp=None):
        """Returns an array of packet table names based on the specified timestamp range

        Note:
            If table does not exist we will not include the packet table name in array

        Args:
            startTimestamp (int):  Start unix timestamp for requested packet tables
            endTimestamp (int):    End unix timestamp for requested packet tables

        Returns:
            Array of packet table names
        """
        if (endTimestamp is None):
            endTimestamp = int(time.time())

        # We allways want to include
        endDateTime = datetime.datetime.utcfromtimestamp(int(endTimestamp))
        endDateTime = endDateTime.replace(
            hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
        endTimestamp = calendar.timegm(endDateTime.timetuple())

        result = []
        if (startTimestamp == 0):
            # Go back 1 year
            ts = int(time.time()) - (60*60*24*366)
        else:
            ts = startTimestamp
        while (ts < endTimestamp):
            date = datetime.datetime.utcfromtimestamp(
                int(ts)).strftime('%Y%m%d')
            datePacketTable = 'packet' + date
            if (self.dbObjectFinder.checkTableExists(datePacketTable)):
                result.append(datePacketTable)

            ts = ts + 86400  # 1 day in seconds
        return result

    def _createPacketTable(self, tablename, minTimestamp, maxTimestamp):
        """Create a packet table with the specified name

        Args:
            tablename (str): Name of the packet table to create
            minTimestamp (int):     Min Unix timestamp for this table
            maxTimestamp (int):     Max Unix timestamp for this table
        """
        try:
            cur = self.db.cursor()
            sql = """
                create table %s () inherits (packet)""" % (tablename)
            cur.execute(sql)

            sql = """alter table %s add constraint timestamp_range_check check(timestamp >= %d and timestamp < %d)""" % (tablename, minTimestamp, maxTimestamp)
            cur.execute(sql)

            # The regular primary key index
            sql = """create index %s_pkey on %s using btree (id)""" % (
                tablename, tablename)
            cur.execute(sql)

            # This index is magic for multiple methods in PacketRepository
            sql = """create index %s_station_id_idx on %s(station_id, map_id, marker_id, timestamp)""" % (
                tablename, tablename)
            cur.execute(sql)

            # This should be good when using the time-travel functionality
            sql = """create index %s_map_sector_idx on %s(map_sector, timestamp, map_id)""" % (
                tablename, tablename)
            cur.execute(sql)

            # Used by remover
            sql = """create index %s_sender_id_idx on %s(sender_id)""" % (
                tablename, tablename)
            cur.execute(sql)

            cur.close()

        except (psycopg2.IntegrityError, psycopg2.ProgrammingError) as e:
            # Probably the other collector created the table at the same time (might happen when you run multiple collectors), just go on...
            if ('already exists' not in str(e)):
                self.logger.error(e, exc_info=1)

            # Do some sleep and let the other process create all related tables (if other table failes we will do it after sleep)
            time.sleep(10)
            return

        except Exception as e:
            self.logger.error(e, exc_info=1)

            # Do some sleep and let the other process create all related tables (if other table failes we will do it after sleep)
            time.sleep(10)
            return