Build System: Enable python3 compatibility.

* Consistently use spaces in rest-api-templates/asterisk_processor.py.
* Exclude third-party from docs/full-en_US.xml.
* Add docs/full-en_US.xml to .gitignore.
* Use list() to convert python3 view.
* Use python3 print function.
* Replace cmp() with equivalent equation.
* Replace reference to out of scope subtype variable with name
  parameter.
* Use unescaping triple bracket notation in mustache templates where
  needed.  This causes behavior of Python2 to be maintained when using
  Python3.
* Fix references to has_websocket / is_websocket in
  res_ari_resource.c.mustache.
* Update calculation of has_websocket to use any().
* Use unicode mode for writing output file in transform.py.
* Replace 'from swagger_model import *' with explicit import of required
  symbols.

I have not tested spandspflow2pcap.py or voicemailpwcheck.py, only the
print syntax has been fixed.

Change-Id: If5c5b556a2800d41a3e2cfef080ac2e151178c33

Makefile

@@ -511,7 +511,7 @@ else
 	@echo "<!DOCTYPE docs SYSTEM \"appdocsxml.dtd\">" >> $@
 	@echo "<?xml-stylesheet type=\"text/xsl\" href=\"appdocsxml.xslt\"?>" >> $@
 	@echo "<docs xmlns:xi=\"http://www.w3.org/2001/XInclude\">" >> $@
-	@for x in $(MOD_SUBDIRS); do \
+	@for x in $(filter-out third-party,$(MOD_SUBDIRS)); do \
 		printf "$$x " ; \
 		for i in `find $$x -name '*.c'`; do \
 			$(PYTHON) build_tools/get_documentation.py < $$i >> $@ ; \

contrib/scripts/refcounter.py

@@ -18,6 +18,7 @@
  Matt Jordan <mjordan@digium.com>
 """
 
+from __future__ import print_function
 import sys
 import os
 
@@ -35,8 +36,8 @@ def parse_line(line):
     """
     tokens = line.strip().split(',', 7)
     if len(tokens) < 8:
-        print "ERROR: ref debug line '%s' contains fewer tokens than " \
-              "expected: %d" % (line.strip(), len(tokens))
+        print("ERROR: ref debug line '%s' contains fewer tokens than "
+              "expected: %d" % (line.strip(), len(tokens)))
         return None
 
     processed_line = {'addr': tokens[0],
@@ -142,7 +143,7 @@ def process_file(options):
                 del current_objects[obj]
 
     if options.leaks:
-        for key, lines in current_objects.iteritems():
+        for (key, lines) in current_objects.items():
             leaked_objects.append((key, lines))
     return (finished_objects, invalid_objects, leaked_objects, skewed_objects)
 
@@ -156,13 +157,13 @@ def print_objects(objects, prefix=""):
             this object
     """
 
-    print "======== %s Objects ========" % prefix
-    print "\n"
+    print("======== %s Objects ========" % prefix)
+    print("\n")
     for obj in objects:
-        print "==== %s Object %s history ====" % (prefix, obj[0])
+        print("==== %s Object %s history ====" % (prefix, obj[0]))
         for line in obj[1]['log']:
-            print line
-        print "\n"
+            print(line)
+        print("\n")
 
 
 def main(argv=None):
@@ -198,11 +199,11 @@ def main(argv=None):
 
     if not options.invalid and not options.leaks and not options.normal \
             and not options.skewed:
-        print >>sys.stderr, "All options disabled"
+        print("All options disabled", file=sys.stderr)
         return -1
 
     if not os.path.isfile(options.filepath):
-        print >>sys.stderr, "File not found: %s" % options.filepath
+        print("File not found: %s" % options.filepath, file=sys.stderr)
         return -1
 
     try:
@@ -227,7 +228,7 @@ def main(argv=None):
             print_objects(finished_objects, "Finalized")
 
     except (KeyboardInterrupt, SystemExit, IOError):
-        print >>sys.stderr, "File processing cancelled"
+        print("File processing cancelled", file=sys.stderr)
         return -1
 
     return ret_code

contrib/scripts/spandspflow2pcap.py

@@ -119,7 +119,7 @@ class FaxPcap(object):
         else:
             self.date += timedelta(microseconds=9000)
 
-        print seqno, '\t', self.date + self.dateoff
+        print(seqno, '\t', self.date + self.dateoff)
 
         # Make packet.
         packet, prev_data = self.data2packet(self.date + self.dateoff,

contrib/scripts/voicemailpwcheck.py

@@ -46,20 +46,20 @@ mailbox, context, old_pw, new_pw = sys.argv[1:5]
 
 # Enforce a password length of at least 6 characters
 if len(new_pw) < REQUIRED_LENGTH:
-    print "INVALID: Password is too short (%d) - must be at least %d" % \
-            (len(new_pw), REQUIRED_LENGTH)
+    print("INVALID: Password is too short (%d) - must be at least %d" % \
+            (len(new_pw), REQUIRED_LENGTH))
     sys.exit(0)
 
 for regex, error in REGEX_BLACKLIST:
     if re.search(regex, new_pw):
-        print "INVALID: %s" % error
+        print("INVALID: %s" % error)
         sys.exit(0)
 
 for pw in PW_BLACKLIST:
     if new_pw.find(pw) != -1:
-        print "INVALID: %s is forbidden in a password" % pw
+        print("INVALID: %s is forbidden in a password" % pw)
         sys.exit(0)
 
-print "VALID"
+print("VALID")
 
 sys.exit(0)

doc/.gitignore

@@ -1,4 +1,5 @@
 core-en_US.xml
+full-en_US.xml
 rest-api
 api
 asterisk-ng-doxygen

rest-api-templates/api.wiki.mustache

@@ -5,7 +5,7 @@ h1. {{name_title}}
 
 {{#apis}}
 {{#operations}}
-| {{http_method}} | [{{wiki_path}}|#{{nickname}}] | {{#response_class}}{{#is_primitive}}{{name}}{{/is_primitive}}{{^is_primitive}}[{{wiki_name}}|{{wiki_prefix}} REST Data Models#{{singular_name}}]{{/is_primitive}}{{/response_class}} | {{summary}} |
+| {{http_method}} | [{{wiki_path}}|#{{nickname}}] | {{#response_class}}{{#is_primitive}}{{name}}{{/is_primitive}}{{^is_primitive}}[{{wiki_name}}|{{wiki_prefix}} REST Data Models#{{singular_name}}]{{/is_primitive}}{{/response_class}} | {{{summary}}} |
 {{/operations}}
 {{/apis}}
 {{#apis}}

rest-api-templates/ari_resource.h.mustache

@@ -76,7 +76,7 @@ int ast_ari_{{c_name}}_{{c_nickname}}_parse_body(
 
 {{/parse_body}}
 /*!
- * \brief {{summary}}
+ * \brief {{{summary}}}
 {{#notes}}
  *
  * {{{notes}}}
@@ -99,7 +99,7 @@ void ast_ari_{{c_name}}_{{c_nickname}}(struct ast_tcptls_session_instance *ser,
 {{#is_websocket}}
 
 /*!
- * \brief {{summary}}
+ * \brief {{{summary}}}
 {{#notes}}
  *
  * {{{notes}}}
@@ -111,7 +111,7 @@ void ast_ari_{{c_name}}_{{c_nickname}}(struct ast_tcptls_session_instance *ser,
 int ast_ari_websocket_{{c_name}}_{{c_nickname}}_init(void);
 
 /*!
- * \brief {{summary}}
+ * \brief {{{summary}}}
 {{#notes}}
  *
  * {{{notes}}}

rest-api-templates/asterisk_processor.py

@@ -23,7 +23,7 @@ Asterisk RESTful HTTP binding code.
 import os
 import re
 
-from swagger_model import *
+from swagger_model import Stringify, SwaggerError, SwaggerPostProcessor
 
 try:
     from collections import OrderedDict
@@ -183,7 +183,7 @@ class AsteriskProcessor(SwaggerPostProcessor):
                 raise SwaggerError(
                     "Should not mix resources in one API declaration", context)
             # root_path isn't needed any more
-            resource_api.root_path = resource_api.root_path.children()[0]
+            resource_api.root_path = list(resource_api.root_path.children())[0]
             if resource_api.name != resource_api.root_path.name:
                 raise SwaggerError(
                     "API declaration name should match", context)
@@ -206,10 +206,10 @@ class AsteriskProcessor(SwaggerPostProcessor):
 
     def process_parameter(self, parameter, context):
         if parameter.param_type == 'body':
-	    parameter.is_body_parameter = True;
+            parameter.is_body_parameter = True;
             parameter.c_data_type = 'struct ast_json *'
         else:
-	    parameter.is_body_parameter = False;
+            parameter.is_body_parameter = False;
             if not parameter.data_type in self.type_mapping:
                 raise SwaggerError(
                     "Invalid parameter type %s" % parameter.data_type, context)

rest-api-templates/make_ari_stubs.py

@@ -16,19 +16,20 @@
 # at the top of the source tree.
 #
 
+from __future__ import print_function
 import sys
 
 try:
     import pystache
 except ImportError:
-    print >> sys.stderr, "Pystache required. Please sudo pip install pystache."
+    print("Pystache required. Please sudo pip install pystache.", file=sys.stderr)
     sys.exit(1)
 
 import os.path
 
 from asterisk_processor import AsteriskProcessor
 from optparse import OptionParser
-from swagger_model import *
+from swagger_model import ResourceListing
 from transform import Transform
 
 TOPDIR = os.path.dirname(os.path.abspath(__file__))

rest-api-templates/res_ari_resource.c.mustache

@@ -55,7 +55,7 @@
 #if defined(AST_DEVMODE)
 #include "ari/ari_model_validators.h"
 #endif
-{{^has_websocket}}
+{{#has_websocket}}
 {{! Only include http_websocket if necessary. Otherwise we'll do a lot of
  *  unnecessary optional_api intialization, which makes optional_api harder
  *  to debug
@@ -278,7 +278,7 @@ static int load_module(void)
 
 {{#apis}}
 {{#operations}}
-{{#has_websocket}}
+{{#is_websocket}}
 	struct ast_websocket_protocol *protocol;
 
 	if (ast_ari_websocket_{{c_name}}_{{c_nickname}}_init() == -1) {
@@ -300,8 +300,6 @@ static int load_module(void)
 	}
 	protocol->session_attempted = ast_ari_{{c_name}}_{{c_nickname}}_ws_attempted_cb;
 	protocol->session_established = ast_ari_{{c_name}}_{{c_nickname}}_ws_established_cb;
-{{/has_websocket}}
-{{#is_websocket}}
 	res |= ast_websocket_server_add_protocol2({{full_name}}.ws_server, protocol);
 {{/is_websocket}}
 {{/operations}}

rest-api-templates/swagger_model.py

@@ -26,6 +26,7 @@ missing, or have incorrect values).
 See https://github.com/wordnik/swagger-core/wiki/API-Declaration for the spec.
 """
 
+from __future__ import print_function
 import json
 import os.path
 import pprint
@@ -75,7 +76,7 @@ def compare_versions(lhs, rhs):
     '''
     lhs = [int(v) for v in lhs.split('.')]
     rhs = [int(v) for v in rhs.split('.')]
-    return cmp(lhs, rhs)
+    return (lhs > rhs) - (lhs < rhs)
 
 
 class ParsingContext(object):
@@ -444,8 +445,7 @@ class Api(Stringify):
         op_json = api_json.get('operations')
         self.operations = [
             Operation().load(j, processor, context) for j in op_json]
-        self.has_websocket = \
-            filter(lambda op: op.is_websocket, self.operations) != []
+        self.has_websocket = any(op.is_websocket for op in self.operations)
         processor.process_api(self, context)
         return self
 
@@ -611,7 +611,7 @@ class ApiDeclaration(Stringify):
         except SwaggerError:
             raise
         except Exception as e:
-            print >> sys.stderr, "Error: ", traceback.format_exc()
+            print("Error: ", traceback.format_exc(), file=sys.stderr)
             raise SwaggerError(
                 "Error loading %s" % api_declaration_file, context, e)
 
@@ -624,8 +624,8 @@ class ApiDeclaration(Stringify):
             .replace(".json", ".{format}")
 
         if self.resource_path != expected_resource_path:
-            print >> sys.stderr, \
-                "%s != %s" % (self.resource_path, expected_resource_path)
+            print("%s != %s" % (self.resource_path, expected_resource_path),
+                file=sys.stderr)
             raise SwaggerError("resourcePath has incorrect value", context)
 
         return self
@@ -656,8 +656,7 @@ class ApiDeclaration(Stringify):
             if api.path in paths:
                 raise SwaggerError("API with duplicated path: %s" % api.path, context)
             paths.add(api.path)
-        self.has_websocket = filter(lambda api: api.has_websocket,
-                                    self.apis) == []
+        self.has_websocket = any(api.has_websocket for api in self.apis)
         models = api_decl_json.get('models').items() or []
         self.models = [Model().load(id, json, processor, context)
                        for (id, json) in models]
@@ -666,7 +665,7 @@ class ApiDeclaration(Stringify):
         model_dict = dict((m.id, m) for m in self.models)
         for m in self.models:
             def link_subtype(name):
-                res = model_dict.get(subtype)
+                res = model_dict.get(name)
                 if not res:
                     raise SwaggerError("%s has non-existing subtype %s",
                                        m.id, name)
@@ -725,7 +724,7 @@ class ResourceListing(Stringify):
         except SwaggerError:
             raise
         except Exception as e:
-            print >> sys.stderr, "Error: ", traceback.format_exc()
+            print("Error: ", traceback.format_exc(), file=sys.stderr)
             raise SwaggerError(
                 "Error loading %s" % resource_file, context, e)
 

rest-api-templates/transform.py

@@ -21,6 +21,11 @@ import os.path
 import pystache
 import shutil
 import tempfile
+import sys
+
+if sys.version_info[0] == 3:
+    def unicode(v):
+        return str(v)
 
 
 class Transform(object):
@@ -52,10 +57,10 @@ class Transform(object):
         dest_exists = os.path.exists(dest_file)
         if dest_exists and not self.overwrite:
             return
-        with tempfile.NamedTemporaryFile() as out:
+        with tempfile.NamedTemporaryFile(mode='w+') as out:
             out.write(renderer.render(self.template, model))
             out.flush()
 
             if not dest_exists or not filecmp.cmp(out.name, dest_file):
-                print "Writing %s" % dest_file
+                print("Writing %s" % dest_file)
                 shutil.copyfile(out.name, dest_file)