make, scripts: Flip some more switches from pathon 2 to 3 (untested!)

This commit flips some more switches from Python 2 to Python 3 in makefiles and
Python code. Build runs through, but it's still likely to break things.

Signed-off-by: Jan Lindemann <jan@janware.com>

make/projects-dir.mk

@@ -17,7 +17,7 @@
 
 -include local.mk
 
-SHELL                        = /bin/bash -o pipefail
+SHELL                        = /bin/bash -o pipefail +H
 PROJECTS_TXT                ?= projects.txt
 JW_BUILD_VERBOSE            ?= false
 BASE_PKGS                    = git cvs make sudo time time xdg-utils
@@ -97,6 +97,11 @@ ifneq ($(EXCLUDE_FROM_BUILD),)
   PROJECTS_PY_EXTRA_BUILD_OPTS += --exclude "$(EXCLUDE_FROM_BUILD)"
 endif
 
+# non-interactive mode
+ifeq ($(shell echo $$PS1),)
+  DASH_Y                    := -y
+endif
+
 # ------------ external programs II
 
 BROWSER                     ?= xdg-open
@@ -104,7 +109,7 @@ EDITOR                      ?= xdg-open
 ifeq ($(TIME),)
   TIME                       = $(shell which time) -p
 endif
-PROJECTS_PY                  = python2 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS)
+PROJECTS_PY                  = python3 $(MOD_SCRIPT_DIR)/projects.py --prefix $(shell pwd) $(PROJECTS_PY_EXTRA_OPTS)
 PROJECTS_PY_BUILD            = $(PROJECTS_PY) build $(PROJECTS_PY_EXTRA_BUILD_OPTS)
 PGIT_SH                      = /bin/bash $(MOD_SCRIPT_DIR)/pgit.sh
 PURGE_SH                     = /bin/bash $(firstword $(wildcard $(MOD_SCRIPT_DIR)/purge-stale-projects.sh /opt/jw-build/bin/purge-stale-projects.sh) purge-not-found)
@@ -205,15 +210,14 @@ cloc:
 	cloc --exclude-list-file=cloc-ignore.txt $(BUILD_PROJECTS)
 
 # --- package-related targets
-
 pkg-manager-refresh:
-	$(PKG_MANAGER_SH) refresh -y
+	$(PKG_MANAGER_SH) refresh $(DASH_Y)
 
 pkg-install-prereq-build:
-	$(PKG_MANAGER_SH) install -y $(BASE_PKGS) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build" $(TARGET_PROJECTS))
+	$(PKG_MANAGER_SH) install $(DASH_Y) $(BASE_PKGS) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build" $(TARGET_PROJECTS))
 
 pkg-install-prereq-release:
-	$(PKG_MANAGER_SH) install -y $(BASE_PKGS) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build run release" $(TARGET_PROJECTS))
+	$(PKG_MANAGER_SH) install $(DASH_Y) $(BASE_PKGS) $(shell $(PROJECTS_PY) requires-pkg --skip-excluded --flavours "build run release" $(TARGET_PROJECTS))
 
 pkg-exclude-built-today:
 	touch $(EXCLUDES_FILE)

make/projects.mk

@@ -21,7 +21,7 @@ else
 endif
 
 # -- Query the build system about other projects:
-PYTHON                      ?= /usr/bin/python2
+PYTHON                      ?= /usr/bin/python3
 ifneq ($(TOPDIR),)
   proj_query_cmd             = $(PYTHON) $(MOD_SCRIPT_DIR)/projects.py -p $(PRJS_DIR) -t $(TOPDIR) $(PROJECTS_PY_EXTRA_ARGS)
   proj_query                 = $(shell $(proj_query_cmd) $(1))

make/py-defs.mk

@@ -1,11 +1,15 @@
 ENV_PYTHONPATH              := $(PYTHONPATH)
-PYTHON                      ?= python2
+ifeq ($(PYTHON),)
-#PYTHON                      = python3
+  PYTHON_VERSION            ?= 3
+  PYTHON                    ?= /usr/bin/python$(PYTHON_VERSION)
+else
+  PYTHON_VERSION            ?= $(patsubst python%,%,$(notdir $(PYTHON)))
+endif
 
 ifneq ($(DEVELOPMENT),true)
   PY_SITE_PACKAGES_PATH     := $(shell $(PYTHON) -c "import site; print(site.getsitepackages()[0])")
 else
-  PY_SITE_PACKAGES_PATH     := $(PREFIX)/python2/site-packages
+  PY_SITE_PACKAGES_PATH     := $(PREFIX)/python$(PYTHON_VERSION)/site-packages
 endif
 
 PY_PREREQ_BUILD             ?= $(shell $(proj_query_cmd) pkg-requires --subsections jw -d ' ' -p --no-version build $(PROJECT))

make/py-run.mk

@@ -12,4 +12,4 @@ clean:
 distclean:
 
 run:
-	/usr/bin/python2 $(EXE) $(EXE_ARGS)
+	$(PYTHON) $(EXE) $(EXE_ARGS)

make/scripts-targets.mk

@@ -11,7 +11,7 @@ endif
 HOME_BIN_EXE_SH = $(addprefix $(HOME)/bin/, $(notdir $(EXE_SH)))
 
 $(HOME)/bin/%.py: %.py
-	echo -e "#!/bin/bash\nexec /usr/bin/python2 $(shell $(PWD))/$<" '"$$@"' > $@.tmp
+	echo -e "#!/bin/bash\nexec $(PYTHON) $(shell $(PWD))/$<" '"$$@"' > $@.tmp
 	chmod 755 $@.tmp
 	mv $@.tmp $@
 

scripts/projects.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2 -u
+#!/usr/bin/python3 -u
 
 from __future__ import print_function
 import os
@@ -88,7 +88,7 @@ class Build(object):
                 prereq_type, scope = 2, add_self=False, names_only=True)
             debug('prerequisites = ' + ' '.join(r))
         else: # legacy from build.py
-            projects_py="/usr/bin/python2 " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
+            projects_py = sys.executable + " " + my_dir + "/projects.py --prefix " + projs_root + " " + os.getenv('PROJECTS_PY_EXTRA_ARGS', "")
             cmd = projects_py + " prereq " + prereq_type + " " + cur
             debug('running', cmd)
             p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
@@ -97,7 +97,8 @@ class Build(object):
                 raise Exception("failed to get " + prereq_type + " prerequisites for " + cur + ": " + cmd)
             r = set()
             pattern = re.compile(r'.*') # might be useful at a later point, currently pointless
-            for line in iter(p.stdout.readline,''):
+            for line in iter(p.stdout.readline, b''):
+                line = line.decode(sys.stdout.encoding)
                 debug(cmd + ' returned: ', line)
                 if not pattern.match(line):
                     continue
@@ -166,7 +167,8 @@ class Build(object):
 
         os.chdir(path)
         p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
-        for line in iter(p.stdout.readline, ''):
+        for line in iter(p.stdout.readline, b''):
+            line = line.decode(sys.stdout.encoding)
             sys.stdout.write('| ' + line) # avoid extra newlines from print()
             sys.stdout.flush()
         p.wait()
@@ -803,7 +805,7 @@ def read_dep_graph(modules, section, graph):
 
 def flip_graph(graph):
     r = {}
-    for m, deps in graph.iteritems():
+    for m, deps in graph.items():
         for d in deps:
             if not d in r:
                 r[d] = set()