Merge branch 'dev', v0.6.37

73306aa - bump to v0.6.37
00bb2f4 - charset detection, unicode for all
bd8b186 - improved _parse_timestamp
829f1fc - fix mic-e dstcall decoding bug
6ec5e1e - tiny syntax fix in parse.py

README.rst

@@ -4,17 +4,24 @@ APRS library for Python
 |Build Status| |Coverage Status|
 
 A tiny library for dealing with APRS. It can be used to connect and listen to the APRS-IS feed as well as upload.
-Parsing of packets is also possible, but the entire spec is not fully implemeneted yet.
+Parsing of packets is also possible, but the entire spec is not fully implemented yet.
 The following is supported:
 
 -  normal/compressed position reports
 -  objects
 -  mic-e position report
 -  messages (inc. telemetry, bulletins, etc)
--  base91 comment telemetry extention
--  altitude extention
+-  base91 comment telemetry extension
+-  altitude extension
 -  beacons
 
+Packets can often contain characters outside of 7-bit ASCII.
+``aprslib.parse()`` will attempt to guess the charset and return ``unicode`` strings using these steps and in that order:
+
+1. Attempt to decode string as ``utf-8``
+2. Attempt to guess the charset using ``chardet`` module (if installed), decode if confidence factor is sufficient
+3. Finally, decode as ``latin-1``
+
 Install
 -----------
 
@@ -116,7 +123,7 @@ Here is a simple example:
     DEBUG:aprslib.parse:Parsed ok.
     ...
 
-Uploading packets to APRS-IS is posible through the ``sendall()`` method in ``IS``.
+Uploading packets to APRS-IS is possible through the ``sendall()`` method in ``IS``.
 The method assumes a single line/packet per call. The parameters may end with ``\r\n``, but it's not required.
 
 .. code:: python

aprslib/__init__.py

@@ -37,7 +37,7 @@ from datetime import date as _date
 __date__ = str(_date.today())
 del _date
 
-__version__ = "0.6.36"
+__version__ = "0.6.37"
 __author__ = "Rossen Georgiev"
 __all__ = ['IS', 'parse', 'passcode']
 

aprslib/base91.py

@@ -29,8 +29,8 @@ def to_decimal(text):
     Takes a base91 char string and returns decimal
     """
 
-    if not isinstance(text, str):
-        raise TypeError("expected str")
+    if not isinstance(text, basestring):
+        raise TypeError("expected str or unicode, %s given" % type(text))
 
     if findall(r"[\x00-\x20\x7c-\xff]", text):
         raise ValueError("invalid character in sequence")

aprslib/parse.py

@@ -24,6 +24,16 @@ import math
 import logging
 from datetime import datetime
 
+try:
+    import chardet
+except ImportError:
+    # create fake chardet
+
+    class chardet:
+        @staticmethod
+        def detect(x):
+            return {'confidence': 0.0, 'encoding': 'windows-1252'}
+
 from .exceptions import (UnknownFormat, ParseError)
 from . import base91
 
@@ -68,6 +78,17 @@ def parse(packet):
       * status message
     """
 
+    # attempt to detect encoding
+    try:
+        packet = packet.decode('utf-8')
+    except UnicodeDecodeError:
+        res = chardet.detect(packet)
+
+        if res['confidence'] > 0.7:
+            packet = packet.decode(res['encoding'])
+        else:
+            packet = packet.decode('latin-1')
+
     packet = packet.rstrip("\r\n")
     logger.debug("Parsing: %s", packet)
 
@@ -93,7 +114,7 @@ def parse(packet):
 
     try:
         parsed.update(_parse_header(head))
-    except ParseError, msg:
+    except ParseError as msg:
         raise ParseError(str(msg), packet)
 
     packet_type = body[0]
@@ -331,7 +352,7 @@ def _parse_timestamp(body, packet_type=''):
 
         if packet_type == '>' and form != 'z':
             pass
-        if form in "hz/":
+        else:
             body = body[7:]
 
             try:
@@ -341,7 +362,9 @@ def _parse_timestamp(body, packet_type=''):
                 # zulu ddhhmm format
                 # '/' local ddhhmm format
                 elif form in 'z/':
-                    timestamp = "%d%02d%s%02d" % (utc.year, utc.month, ts, utc.second)
+                    timestamp = "%d%02d%s%02d" % (utc.year, utc.month, ts, 0)
+                else:
+                    timestamp = "19700101000000"
 
                 timestamp = utc.strptime(timestamp, "%Y%m%d%H%M%S")
                 timestamp = time.mktime(timestamp.timetuple())
@@ -477,7 +500,7 @@ def _parse_mice(dstcall, body):
         elif ord(i) > 76:  # P-Y
             tmpdstcall += chr(ord(i) - 32)
         elif ord(i) > 57:  # A-J
-            tmpdstcall += chr(ord(i) - 16)
+            tmpdstcall += chr(ord(i) - 17)
         else:  # 0-9
             tmpdstcall += i
 

tests/test_base91.py

@@ -64,12 +64,13 @@ class b_ToDecimal(unittest.TestCase):
         # 91**3 = "!!!
         # etc
         testData += [[91**i, '"' + '!'*i] for i in xrange(20)]
+        testData += [[91**i, u'"' + u'!'*i] for i in xrange(20)]
 
         for expected, n in testData:
             self.assertEqual(expected, base91.to_decimal(n))
 
     def test_invalid_input_type(self):
-        testData = [-1, 0, 5, unicode('a'), None, ['d']]
+        testData = [-1, 0, 5, None, ['d']]
 
         for n in testData:
             self.assertRaises(TypeError, base91.to_decimal, n)