Migrate over to using pathlib and subprocess.run()

src/funfuzz/autobisectjs/known_broken_earliest_working.py

@@ -129,7 +129,8 @@ def earliest_known_working_rev(options, flags, skip_revs):  # pylint: disable=mi
     # Note that the sed version check only works with GNU sed, not BSD sed found in macOS.
     if (platform.system() == "Linux" and
             parse_version(subprocess.run(["sed", "--version"],
-                                         stdout=subprocess.PIPE).stdout.split()[3]) >= parse_version("4.3")):
+                                         stdout=subprocess.PIPE).stdout.decode("utf-8", errors="replace").split()[3])
+            >= parse_version("4.3")):
         required.append("ebcbf47a83e7")  # m-c 328765 Fx53, 1st w/ working builds using sed 4.3+ found on Ubuntu 17.04+
     if options.disableProfiling:
         required.append("800a887c705e")  # m-c 324836 Fx53, 1st w/ --disable-profiling, see bug 1321065

src/funfuzz/bot.py

@@ -26,13 +26,17 @@
 from .util import create_collector
 from .util import fork_join
 from .util import hg_helpers
-from .util import subprocesses as sps
 from .util.lock_dir import LockDir
 
 if sys.version_info.major == 2:
+    from pathlib2 import Path
     import psutil
+    if os.name == "posix":
+        import subprocess32 as subprocess  # pylint: disable=import-error
+else:
+    import subprocess
+    from pathlib import Path  # pylint: disable=import-error
 
-path0 = os.path.dirname(os.path.abspath(__file__))  # pylint: disable=invalid-name
 JS_SHELL_DEFAULT_TIMEOUT = 24  # see comments in loop for tradeoffs
 
 
@@ -84,7 +88,8 @@ def parseOpts():  # pylint: disable=invalid-name,missing-docstring,missing-retur
     if args:
         print("Warning: bot does not use positional arguments")
 
-    if not options.useTreeherderBuilds and not os.path.isdir(build_options.DEFAULT_TREES_LOCATION):
+    # pylint: disable=no-member
+    if not options.useTreeherderBuilds and not build_options.DEFAULT_TREES_LOCATION.is_dir():
         # We don't have trees, so we must use treeherder builds.
         options.useTreeherderBuilds = True
         print()
@@ -106,7 +111,7 @@ def main():  # pylint: disable=missing-docstring
 
     options = parseOpts()
 
-    collector = create_collector.createCollector("jsfunfuzz")
+    collector = create_collector.make_collector()
     try:
         collector.refresh()
     except RuntimeError as ex:
@@ -118,10 +123,10 @@ def main():  # pylint: disable=missing-docstring
     print(options.tempDir)
 
     build_info = ensureBuild(options)
-    assert os.path.isdir(build_info.buildDir)
+    assert build_info.buildDir.is_dir()
 
     number_of_processes = multiprocessing.cpu_count()
-    if "-asan" in build_info.buildDir:
+    if "-asan" in str(build_info.buildDir):
         # This should really be based on the amount of RAM available, but I don't know how to compute that in Python.
         # I could guess 1 GB RAM per core, but that wanders into sketchyville.
         number_of_processes = max(number_of_processes // 2, 1)
@@ -134,18 +139,24 @@ def main():  # pylint: disable=missing-docstring
 def printMachineInfo():  # pylint: disable=invalid-name
     """Log information about the machine."""
     print("Platform details: %s" % " ".join(platform.uname()))
-    print("hg version: %s" % sps.captureStdout(["hg", "-q", "version"])[0])
-
-    # In here temporarily to see if mock Linux slaves on TBPL have gdb installed
+    print("hg version: %s" %
+          subprocess.run(["hg", "-q", "version"],
+                         check=True,
+                         cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(),  # pylint: disable=no-member
+                         stdout=subprocess.PIPE,
+                         timeout=9).stdout.rstrip())
     try:
-        print("gdb version: %s" % sps.captureStdout(["gdb", "--version"], combineStderr=True,
-                                                    ignoreStderr=True, ignoreExitCode=True)[0])
-    except (KeyboardInterrupt, Exception) as ex:  # pylint: disable=broad-except
+        print("gdb version: %s" %
+              subprocess.run(
+                  ["gdb", "--version"],
+                  cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(),  # pylint: disable=no-member
+                  stderr=subprocess.STDOUT,
+                  stdout=subprocess.PIPE,
+                  timeout=9).stdout.decode("utf-8", errors="replace").split("\n")[0])
+    except (KeyboardInterrupt, subprocess.CalledProcessError) as ex:
         print("Error involving gdb is: %r" % (ex,))
 
-    # FIXME: Should have if os.path.exists(path to git) or something  # pylint: disable=fixme
-    # print("git version: %s" % sps.captureStdout(["git", "--version"], combineStderr=True,
-    #                                             ignoreStderr=True, ignoreExitCode=True)[0])
+    # FIXME: Should have if which(git) or something  # pylint: disable=fixme
     print("Python version: %s" % sys.version.split()[0])
     print("Number of cores visible to OS: %d" % multiprocessing.cpu_count())
     if sys.version_info.major == 2:
@@ -154,18 +165,20 @@ def printMachineInfo():  # pylint: disable=invalid-name
         rootdir_free_space = shutil.disk_usage("/").free / (1024 ** 3)  # pylint: disable=no-member
     print("Free space (GB): %.2f" % rootdir_free_space)
 
-    hgrc_path = os.path.join(path0, ".hg", "hgrc")
-    if os.path.isfile(hgrc_path):
+    hgrc_path = Path("~/.hg/hgrc").expanduser()
+    if hgrc_path.is_file():
         print("The hgrc of this repository is:")
-        with open(hgrc_path, "r") as f:
+        with open(str(hgrc_path), "r") as f:
             hgrc_contents = f.readlines()
         for line in hgrc_contents:
             print(line.rstrip())
 
-    if os.name == "posix":
+    try:
         # resource library is only applicable to Linux or Mac platforms.
         import resource  # pylint: disable=import-error
         print("Corefile size (soft limit, hard limit) is: %r" % (resource.getrlimit(resource.RLIMIT_CORE),))
+    except ImportError:
+        print("Not checking corefile size as resource module is unavailable")
 
 
 def ensureBuild(options):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
@@ -177,16 +190,16 @@ def ensureBuild(options):  # pylint: disable=invalid-name,missing-docstring,miss
         bRev = ""  # pylint: disable=invalid-name
         manyTimedRunArgs = []  # pylint: disable=invalid-name
     elif not options.useTreeherderBuilds:
-        options.build_options = build_options.parseShellOptions(options.build_options)
+        options.build_options = build_options.parse_shell_opts(options.build_options)
         options.timeout = options.timeout or (300 if options.build_options.runWithVg else JS_SHELL_DEFAULT_TIMEOUT)
 
-        with LockDir(compile_shell.getLockDirPath(options.build_options.repoDir)):
-            bRev = hg_helpers.getRepoHashAndId(options.build_options.repoDir)[0]  # pylint: disable=invalid-name
+        with LockDir(compile_shell.get_lock_dir_path(Path.home(), options.build_options.repo_dir)):
+            bRev = hg_helpers.get_repo_hash_and_id(options.build_options.repo_dir)[0]  # pylint: disable=invalid-name
             cshell = compile_shell.CompiledShell(options.build_options, bRev)
-            updateLatestTxt = (options.build_options.repoDir == "mozilla-central")  # pylint: disable=invalid-name
+            updateLatestTxt = (options.build_options.repo_dir == "mozilla-central")  # pylint: disable=invalid-name
             compile_shell.obtainShell(cshell, updateLatestTxt=updateLatestTxt)
 
-            bDir = cshell.getShellCacheDir()  # pylint: disable=invalid-name
+            bDir = cshell.get_shell_cache_dir()  # pylint: disable=invalid-name
             # Strip out first 3 chars or else the dir name in fuzzing jobs becomes:
             #   js-js-dbg-opt-64-dm-linux
             bType = build_options.computeShellType(options.build_options)[3:]  # pylint: disable=invalid-name
@@ -199,7 +212,7 @@ def ensureBuild(options):  # pylint: disable=invalid-name,missing-docstring,miss
                 "==============================================\n\n" % (
                     "funfuzz.js.compile_shell",
                     options.build_options.build_options_str,
-                    options.build_options.repoDir,
+                    options.build_options.repo_dir,
                     bRev,
                     cshell.getRepoName(),
                     time.asctime()
@@ -216,15 +229,15 @@ def ensureBuild(options):  # pylint: disable=invalid-name,missing-docstring,miss
 
 
 def loopFuzzingAndReduction(options, buildInfo, collector, i):  # pylint: disable=invalid-name,missing-docstring
-    tempDir = tempfile.mkdtemp("loop" + str(i))  # pylint: disable=invalid-name
+    tempDir = Path(tempfile.mkdtemp("loop" + str(i)))  # pylint: disable=invalid-name
     loop.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs, collector)
 
 
 def mtrArgsCreation(options, cshell):  # pylint: disable=invalid-name,missing-param-doc,missing-return-doc
     # pylint: disable=missing-return-type-doc,missing-type-doc
     """Create many_timed_run arguments for compiled builds."""
     manyTimedRunArgs = []  # pylint: disable=invalid-name
-    manyTimedRunArgs.append("--repo=" + sps.normExpUserPath(options.build_options.repoDir))
+    manyTimedRunArgs.append("--repo=%s" % options.build_options.repo_dir)
     manyTimedRunArgs.append("--build=" + options.build_options.build_options_str)
     if options.build_options.runWithVg:
         manyTimedRunArgs.append("--valgrind")
@@ -237,9 +250,5 @@ def mtrArgsCreation(options, cshell):  # pylint: disable=invalid-name,missing-pa
     # Ordering of elements in manyTimedRunArgs is important.
     manyTimedRunArgs.append(str(options.timeout))
     manyTimedRunArgs.append(cshell.getRepoName())  # known bugs' directory
-    manyTimedRunArgs.append(cshell.getShellCacheFullPath())
+    manyTimedRunArgs.append(cshell.get_shell_cache_js_bin_path())
     return manyTimedRunArgs
-
-
-if __name__ == "__main__":
-    main()

src/funfuzz/ccoverage/get_build.py

@@ -11,12 +11,17 @@
 
 import io
 import logging
-import os
+import sys
 import tarfile
 import zipfile
 
 import requests
 
+if sys.version_info.major == 2:
+    from pathlib2 import Path
+else:
+    from pathlib import Path  # pylint: disable=import-error
+
 RUN_COV_LOG = logging.getLogger("funfuzz")
 
 
@@ -41,7 +46,8 @@ def get_coverage_build(dirpath, args):
 
     js_cov_bin_name = "js"
     js_cov_bin = extract_folder / "dist" / "bin" / js_cov_bin_name
-    os.chmod(str(js_cov_bin), os.stat(str(js_cov_bin)).st_mode | 0o111)  # Ensure the js binary is executable
+
+    Path.chmod(js_cov_bin, Path.stat(js_cov_bin).st_mode | 0o111)  # Ensure the js binary is executable
     assert js_cov_bin.is_file()
 
     # Check that the binary is non-debug.

src/funfuzz/js/__init__.py

@@ -13,5 +13,6 @@
 from . import compile_shell
 from . import inspect_shell
 from . import js_interesting
+from . import link_fuzzer
 from . import loop
 from . import shell_flags

src/funfuzz/js/build_options.py

@@ -12,17 +12,20 @@
 import argparse
 from builtins import object  # pylint: disable=redefined-builtin
 import hashlib
-import os
 import platform
 import random
 import sys
 
 from past.builtins import range
 
 from ..util import hg_helpers
-from ..util import subprocesses as sps
 
-DEFAULT_TREES_LOCATION = sps.normExpUserPath(os.path.join("~", "trees"))
+if sys.version_info.major == 2:
+    from pathlib2 import Path
+else:
+    from pathlib import Path  # pylint: disable=import-error
+
+DEFAULT_TREES_LOCATION = Path.home() / "trees"
 
 
 def chance(p):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
@@ -67,11 +70,13 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs):  # pylint
                         action="store_true",
                         default=False,
                         help='Chooses sensible random build options. Defaults to "%(default)s".')
-    parser.add_argument("-R", "--repoDir",
-                        dest="repoDir",
+    parser.add_argument("-R", "--repodir",
+                        dest="repo_dir",
+                        type=Path,
                         help="Sets the source repository.")
     parser.add_argument("-P", "--patch",
-                        dest="patchFile",
+                        dest="patch_file",
+                        type=Path,
                         help="Define the path to a single JS patch. Ensure mq is installed.")
 
     # Basic spidermonkey options
@@ -156,11 +161,17 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs):  # pylint
     return parser, randomizer
 
 
-def parseShellOptions(inputArgs):  # pylint: disable=invalid-name,missing-param-doc,missing-return-doc
-    # pylint: disable=missing-return-type-doc,missing-type-doc
-    """Return a "build_options" object, which is intended to be immutable."""
+def parse_shell_opts(args):
+    """Parses shell options into a build_options object.
+
+    Args:
+        args (object): Arguments to be parsed
+
+    Returns:
+        build_options: An immutable build_options object
+    """
     parser, randomizer = addParserOptions()
-    build_options = parser.parse_args(inputArgs.split())
+    build_options = parser.parse_args(args.split())
 
     if platform.system() == "Darwin":
         build_options.buildWithClang = True  # Clang seems to be the only supported compiler
@@ -171,31 +182,29 @@ def parseShellOptions(inputArgs):  # pylint: disable=invalid-name,missing-param-
     if build_options.enableRandom:
         build_options = generateRandomConfigurations(parser, randomizer)
     else:
-        build_options.build_options_str = inputArgs
+        build_options.build_options_str = args
         valid = areArgsValid(build_options)
         if not valid[0]:
             print("WARNING: This set of build options is not tested well because: %s" % valid[1])
 
     # Ensures releng machines do not enter the if block and assumes mozilla-central always exists
-    if os.path.isdir(DEFAULT_TREES_LOCATION):
+    if DEFAULT_TREES_LOCATION.is_dir():  # pylint: disable=no-member
         # Repositories do not get randomized if a repository is specified.
-        if build_options.repoDir is None:
+        if build_options.repo_dir is None:
             # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead
-            if build_options.enableRandom and not build_options.patchFile:
-                build_options.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION)
+            if build_options.enableRandom and not build_options.patch_file:
+                build_options.repo_dir = get_random_valid_repo(DEFAULT_TREES_LOCATION)
             else:
-                build_options.repoDir = os.path.realpath(sps.normExpUserPath(
-                    os.path.join(DEFAULT_TREES_LOCATION, "mozilla-central")))
+                build_options.repo_dir = DEFAULT_TREES_LOCATION / "mozilla-central"
 
-            if not os.path.isdir(build_options.repoDir):
-                sys.exit("repoDir is not specified, and a default repository location cannot be confirmed. Exiting...")
+            if not build_options.repo_dir.is_dir():
+                sys.exit("repo_dir is not specified, and a default repository location cannot be confirmed. Exiting...")
 
-        assert hg_helpers.isRepoValid(build_options.repoDir)
+        assert (build_options.repo_dir / ".hg" / "hgrc").is_file()
 
-        if build_options.patchFile:
-            hg_helpers.ensureMqEnabled()
-            build_options.patchFile = sps.normExpUserPath(build_options.patchFile)
-            assert os.path.isfile(build_options.patchFile)
+        if build_options.patch_file:
+            hg_helpers.ensure_mq_enabled()
+            assert build_options.patch_file.resolve().is_file()
     else:
         sys.exit("DEFAULT_TREES_LOCATION not found at: %s. Exiting..." % DEFAULT_TREES_LOCATION)
 
@@ -230,10 +239,10 @@ def computeShellType(build_options):  # pylint: disable=invalid-name,missing-par
     if build_options.enableSimulatorArm32 or build_options.enableSimulatorArm64:
         fileName.append("armSim")
     fileName.append("windows" if platform.system() == "Windows" else platform.system().lower())
-    if build_options.patchFile:
+    if build_options.patch_file:
         # We take the name before the first dot, so Windows (hopefully) does not get confused.
-        fileName.append(os.path.basename(build_options.patchFile).split(".")[0])
-        with open(os.path.abspath(build_options.patchFile), "r") as f:
+        fileName.append(build_options.patch_file.name)
+        with open(str(build_options.patch_file.resolve()), "r") as f:
             readResult = f.read()  # pylint: disable=invalid-name
         # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch.
         fileName.append(hashlib.sha512(readResult).hexdigest()[:12])
@@ -332,20 +341,28 @@ def generateRandomConfigurations(parser, randomizer):  # pylint: disable=invalid
             return build_options
 
 
-def getRandomValidRepo(treeLocation):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
-    # pylint: disable=missing-return-type-doc
-    validRepos = []  # pylint: disable=invalid-name
-    for repo in ["mozilla-central", "mozilla-beta"]:
-        if os.path.isfile(sps.normExpUserPath(os.path.join(
-                treeLocation, repo, ".hg", "hgrc"))):
-            validRepos.append(repo)
+def get_random_valid_repo(tree):
+    """Given a path to Mozilla Mercurial repositories, return a randomly chosen valid one.
+
+    Args:
+        tree (str): Intended location of Mozilla Mercurial repositories
+
+    Returns:
+        Path: Location of a valid Mozilla repository
+    """
+    assert isinstance(tree, Path)  # We can remove casting Path to str after moving to Python 3.6+ completely
+    tree = tree.resolve()
+
+    valid_repos = []
+    for _ in ["mozilla-central", "mozilla-beta"]:
+        if (tree / _ / ".hg" / "hgrc").is_file():
+            valid_repos.append(_)
 
     # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
-    if "mozilla-beta" in validRepos and chance(0.5):
-        validRepos.remove("mozilla-beta")
+    if "mozilla-beta" in valid_repos and chance(0.5):
+        valid_repos.remove("mozilla-beta")
 
-    return os.path.realpath(sps.normExpUserPath(
-        os.path.join(treeLocation, random.choice(validRepos))))
+    return tree / random.choice(valid_repos)
 
 
 def main():  # pylint: disable=missing-docstring
@@ -363,7 +380,7 @@ def main():  # pylint: disable=missing-docstring
     print()
     print("Running this file directly doesn't do anything, but here's our subparser help:")
     print()
-    parseShellOptions("--help")
+    parse_shell_opts("--help")
 
 
 if __name__ == "__main__":