From d3f6ffb16486e4d898d6729071f2b327e9a443f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9Fmur=20Oymak?= Date: Fri, 12 Jul 2019 13:27:51 +0300 Subject: [PATCH 1/5] Make build.py compatible with Python 2 and 3 Port of https://github.com/google/blockly/pull/2123 (by @gomercin) --- build.py | 21 +++++++++++++-------- i18n/common.py | 8 ++++---- i18n/create_messages.py | 10 +++++----- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/build.py b/build.py index b560599611..9bd0f1b8e2 100755 --- a/build.py +++ b/build.py @@ -35,11 +35,16 @@ # msg/js/.js for every language defined in msg/js/.json. import sys -if sys.version_info[0] != 2: - raise Exception("Blockly build only compatible with Python 2.x.\n" - "You are using: " + sys.version) -import errno, glob, httplib, json, os, re, subprocess, threading, urllib +import errno, glob, json, os, re, subprocess, threading, codecs + +if sys.version_info[0] == 2: + import httplib + from urllib import urlencode +else: + import http.client as httplib + from urllib.parse import urlencode + from importlib import reload REMOTE_COMPILER = "remote" @@ -373,7 +378,7 @@ def do_compile_remote(self, params, target_filename): headers = {"Content-type": "application/x-www-form-urlencoded"} conn = httplib.HTTPSConnection("closure-compiler.appspot.com") - conn.request("POST", "/compile", urllib.urlencode(remoteParams), headers) + conn.request("POST", "/compile", urlencode(remoteParams), headers) response = conn.getresponse() json_str = response.read() conn.close() @@ -388,12 +393,12 @@ def file_lookup(name): n = int(name[6:]) - 1 return filenames[n] - if json_data.has_key("serverErrors"): + if "serverErrors" in json_data: errors = json_data["serverErrors"] for error in errors: print("SERVER ERROR: %s" % target_filename) print(error["error"]) - elif json_data.has_key("errors"): + elif "errors" in json_data: errors = json_data["errors"] for error in errors: print("FATAL ERROR") @@ -405,7 +410,7 @@ def file_lookup(name): print((" " * error["charno"]) + "^") sys.exit(1) else: - if json_data.has_key("warnings"): + if "warnings" in json_data: warnings = json_data["warnings"] for warning in warnings: print("WARNING") diff --git a/i18n/common.py b/i18n/common.py index 90e584e160..2323cea0ec 100644 --- a/i18n/common.py +++ b/i18n/common.py @@ -59,7 +59,7 @@ def read_json_file(filename): if '@metadata' in defs: del defs['@metadata'] return defs - except ValueError, e: + except ValueError as e: print('Error reading ' + filename) raise InputError(filename, str(e)) @@ -85,7 +85,7 @@ def _create_qqq_file(output_dir): """ qqq_file_name = os.path.join(os.curdir, output_dir, 'qqq.json') qqq_file = codecs.open(qqq_file_name, 'w', 'utf-8') - print 'Created file: ' + qqq_file_name + print('Created file: ' + qqq_file_name) qqq_file.write('{\n') return qqq_file @@ -126,7 +126,7 @@ def _create_lang_file(author, lang, output_dir): """ lang_file_name = os.path.join(os.curdir, output_dir, lang + '.json') lang_file = codecs.open(lang_file_name, 'w', 'utf-8') - print 'Created file: ' + lang_file_name + print('Created file: ' + lang_file_name) # string.format doesn't like printing braces, so break up our writes. lang_file.write('{\n\t"@metadata": {') lang_file.write(""" @@ -166,7 +166,7 @@ def _create_key_file(output_dir): key_file_name = os.path.join(os.curdir, output_dir, 'keys.json') key_file = open(key_file_name, 'w') key_file.write('{\n') - print 'Created file: ' + key_file_name + print('Created file: ' + key_file_name) return key_file diff --git a/i18n/create_messages.py b/i18n/create_messages.py index dc2620a314..c83af54b33 100755 --- a/i18n/create_messages.py +++ b/i18n/create_messages.py @@ -30,8 +30,8 @@ def string_is_ascii(s): try: - s.decode('ascii') - return True + # This approach is better for compatibility + return all(ord(c) < 128 for c in s) except UnicodeEncodeError: return False @@ -81,14 +81,14 @@ def main(): print('ERROR: definition of {0} in {1} contained a newline character.'. format(key, args.source_lang_file)) sys.exit(1) - sorted_keys = source_defs.keys() - sorted_keys.sort() + sorted_keys = sorted(source_defs.keys()) # Read in synonyms file, which must be output in every language. synonym_defs = read_json_file(os.path.join( os.curdir, args.source_synonym_file)) + # synonym_defs is also being sorted to ensure the same order is kept synonym_text = '\n'.join(['Blockly.Msg.{0} = Blockly.Msg.{1};'.format( - key, synonym_defs[key]) for key in synonym_defs]) + key, synonym_defs[key]) for key in sorted(synonym_defs)]) # Read in constants file, which must be output in every language. constants_text = load_constants(os.path.join(os.curdir, args.source_constants_file)) From b93da0bfa7998a7357f1fbaa78f960463492bb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9Fmur=20Oymak?= Date: Fri, 12 Jul 2019 13:45:39 +0300 Subject: [PATCH 2/5] Fix issues related to Python 3 compatibility Port of https://github.com/google/blockly/pull/2125 (by @gomercin) --- build.py | 5 +++-- i18n/dedup_json.py | 4 ++-- i18n/json_to_js.py | 2 +- i18n/tests.py | 2 +- i18n/xliff_to_json.py | 10 +++++----- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/build.py b/build.py index 9bd0f1b8e2..9a1ba454f7 100755 --- a/build.py +++ b/build.py @@ -380,7 +380,8 @@ def do_compile_remote(self, params, target_filename): conn = httplib.HTTPSConnection("closure-compiler.appspot.com") conn.request("POST", "/compile", urlencode(remoteParams), headers) response = conn.getresponse() - json_str = response.read() + # Decode is necessary for Python 3.4 compatibility + json_str = response.read().decode("utf-8") conn.close() # Parse the JSON response. @@ -505,7 +506,7 @@ def _rebuild(self, srcs, dests): # If a destination file was missing, rebuild. return True else: - print("Error checking file creation times: " + e) + print("Error checking file creation times: " + str(e)) def run(self): # The files msg/json/{en,qqq,synonyms}.json depend on msg/messages.js. diff --git a/i18n/dedup_json.py b/i18n/dedup_json.py index 30e572dde9..a27df50f72 100755 --- a/i18n/dedup_json.py +++ b/i18n/dedup_json.py @@ -51,9 +51,9 @@ def main(): try: with codecs.open(filename, 'r', 'utf-8') as infile: j = json.load(infile) - except ValueError, e: + except ValueError as e: print('Error reading ' + filename) - raise InputError(file, str(e)) + raise InputError(filename, str(e)) # Built up output strings as an array to make output of delimiters easier. output = [] diff --git a/i18n/json_to_js.py b/i18n/json_to_js.py index f8c20f6af9..bf3fb38df5 100755 --- a/i18n/json_to_js.py +++ b/i18n/json_to_js.py @@ -100,7 +100,7 @@ def _process_file(path_to_json, target_lang, key_dict): if key != '@metadata': try: identifier = key_dict[key] - except KeyError, e: + except KeyError as e: print('Key "%s" is in %s but not in %s' % (key, keyfile, args.key_file)) raise e diff --git a/i18n/tests.py b/i18n/tests.py index 7e4fc49aa3..2de6fef601 100644 --- a/i18n/tests.py +++ b/i18n/tests.py @@ -37,7 +37,7 @@ def contains_all_chars(orig, result): u'block of actions.'] for sentence in sentences: output = common.insert_breaks(sentence, 30, 50) - self.assert_(contains_all_chars(sentence, output), + self.assertTrue(contains_all_chars(sentence, output), u'Mismatch between:\n{0}\n{1}'.format( re.sub(spaces, '', sentence), re.sub(spaces, '', output))) diff --git a/i18n/xliff_to_json.py b/i18n/xliff_to_json.py index b38b4d6eca..c95d83366f 100755 --- a/i18n/xliff_to_json.py +++ b/i18n/xliff_to_json.py @@ -65,7 +65,7 @@ def get_value(tag_name): try: result['source'] = get_value('source') result['target'] = get_value('target') - except InputError, e: + except InputError as e: raise InputError(key, e.msg) # Get notes, using the from value as key and the data as value. @@ -112,8 +112,8 @@ def _process_file(filename): except IOError: # Don't get caught by below handler raise - except Exception, e: - print + except Exception as e: + print() raise InputError(filename, str(e)) # Make sure needed fields are present and non-empty. @@ -146,8 +146,8 @@ def _process_file(filename): results.append(unit) return results - except IOError, e: - print 'Error with file {0}: {1}'.format(filename, e.strerror) + except IOError as e: + print('Error with file {0}: {1}'.format(filename, e.strerror)) sys.exit(1) From 9a62c8be370cd353474bd6a27ba2eec7b46e2918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9Fmur=20Oymak?= Date: Fri, 12 Jul 2019 14:38:30 +0300 Subject: [PATCH 3/5] Fix compatibility problems with Python 3's reduce(), filter() et al. --- build.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build.py b/build.py index 9a1ba454f7..5f262dfe8b 100755 --- a/build.py +++ b/build.py @@ -36,7 +36,7 @@ import sys -import errno, glob, json, os, re, subprocess, threading, codecs +import errno, glob, json, os, re, subprocess, threading, codecs, functools if sys.version_info[0] == 2: import httplib @@ -199,7 +199,7 @@ def format_js(self, code): key_whitelist = self.closure_env.keys() - keys_pipe_separated = reduce(lambda accum, key: accum + "|" + key, key_whitelist) + keys_pipe_separated = functools.reduce(lambda accum, key: accum + "|" + key, key_whitelist) begin_brace = re.compile(r"\{(?!%s)" % (keys_pipe_separated,)) end_brace = re.compile(r"\}") @@ -341,7 +341,7 @@ def do_compile_local(self, params, target_filename): return dict( compiledCode=stdout, statistics=dict( - originalSize=reduce(lambda v, size: v + size, filesizes, 0), + originalSize=functools.reduce(lambda v, size: v + size, filesizes, 0), compressedSize=len(stdout), ) ) @@ -428,7 +428,7 @@ def file_lookup(name): return False def write_output(self, target_filename, remove, json_data): - if not json_data.has_key("compiledCode"): + if "compiledCode" not in json_data: print("FATAL ERROR: Compiler did not return compiledCode.") sys.exit(1) @@ -608,11 +608,11 @@ def exclude_horizontal(item): developers.google.com/blockly/guides/modify/web/closure""") sys.exit(1) - search_paths = calcdeps.ExpandDirectories( - ["core", os.path.join(closure_root, closure_library)]) + search_paths = list(calcdeps.ExpandDirectories( + ["core", os.path.join(closure_root, closure_library)])) - search_paths_horizontal = filter(exclude_vertical, search_paths) - search_paths_vertical = filter(exclude_horizontal, search_paths) + search_paths_horizontal = list(filter(exclude_vertical, search_paths)) + search_paths_vertical = list(filter(exclude_horizontal, search_paths)) closure_env = { "closure_dir": closure_dir, From 3f8ddc0de77054f3e7e99e6c0ac5d8544556d856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9Fmur=20Oymak?= Date: Fri, 12 Jul 2019 15:19:18 +0300 Subject: [PATCH 4/5] Fix Python 3 compatibility problems related to str encodings --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 5f262dfe8b..3910b4576b 100755 --- a/build.py +++ b/build.py @@ -432,7 +432,7 @@ def write_output(self, target_filename, remove, json_data): print("FATAL ERROR: Compiler did not return compiledCode.") sys.exit(1) - code = HEADER + "\n" + json_data["compiledCode"] + code = HEADER + "\n" + json_data["compiledCode"].decode("utf-8") code = code.replace(remove, "") # Trim down Google's (and only Google's) Apache licences. @@ -579,7 +579,7 @@ def exclude_horizontal(item): test_args = [closure_compiler, os.path.join("build", "test_input.js")] test_proc = subprocess.Popen(test_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (stdout, _) = test_proc.communicate() - assert stdout == read(os.path.join("build", "test_expect.js")) + assert stdout.decode("utf-8") == read(os.path.join("build", "test_expect.js")) print("Using local compiler: google-closure-compiler ...\n") except (ImportError, AssertionError): From 1a33ef89c2fe988ce9ff0ed751d898b8d667168c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9Fmur=20Oymak?= Date: Tue, 30 Jun 2020 11:19:00 +0300 Subject: [PATCH 5/5] Remove obsolete UnicodeEncodeError handling --- i18n/create_messages.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/i18n/create_messages.py b/i18n/create_messages.py index c83af54b33..2d37f25c1b 100755 --- a/i18n/create_messages.py +++ b/i18n/create_messages.py @@ -29,11 +29,8 @@ def string_is_ascii(s): - try: - # This approach is better for compatibility - return all(ord(c) < 128 for c in s) - except UnicodeEncodeError: - return False + # This approach is better for compatibility + return all(ord(c) < 128 for c in s) def load_constants(filename): """Read in constants file, which must be output in every language."""