[3.6] bpo-23404: make touch becomes make regen-all (#1405) (#1461)

* bpo-23404: make touch becomes make regen-all (#1405)

Don't rebuild generated files based on file modification time
anymore, the action is now explicit. Replace "make touch"
with "make regen-all".

Changes:

* Remove "make touch", Tools/hg/hgtouch.py and .hgtouch
* Add a new "make regen-all" command to rebuild all generated files
* Add subcommands to only generate specific files:

  - regen-ast: Include/Python-ast.h and Python/Python-ast.c
  - regen-grammar: Include/graminit.h and Python/graminit.c
  - regen-importlib: Python/importlib_external.h and Python/importlib.h
  - regen-opcode: Include/opcode.h
  - regen-opcode-targets: Python/opcode_targets.h
  - regen-typeslots: Objects/typeslots.inc

* Rename PYTHON_FOR_GEN to PYTHON_FOR_REGEN
* pgen is now only built by by "make regen-grammar"
* Add $(srcdir)/ prefix to paths to source files to handle correctly
  compilation outside the source directory

Note: $(PYTHON_FOR_REGEN) is no more used nor needed by "make"
default target building Python.
(cherry picked from commit a5c62a8e9f)

* bpo-30273: Update sysconfig (#1464)

The AST_H_DIR variable was removed from Makefile.pre.in by the commit
a5c62a8e9f (bpo-23404).

AST_H_DIR was hardcoded to "Include", so replace the removed variable
by its content.

Remove also ASDLGEN variable from sysconfig example since this
variable was also removed.
(cherry picked from commit b109a1d336)

Tools/hg/hgtouch.py

@@ -1,129 +0,0 @@
-"""Bring time stamps of generated checked-in files into the right order
-
-A versioned configuration file .hgtouch specifies generated files, in the
-syntax of make rules.
-
-  output:    input1 input2
-
-In addition to the dependency syntax, #-comments are supported.
-"""
-import errno
-import os
-import time
-
-def parse_config(repo):
-    try:
-        fp = repo.wfile(".hgtouch")
-    except IOError, e:
-        if e.errno != errno.ENOENT:
-            raise
-        return {}
-    result = {}
-    with fp:
-        for line in fp:
-            # strip comments
-            line = line.split('#')[0].strip()
-            if ':' not in line:
-                continue
-            outputs, inputs = line.split(':', 1)
-            outputs = outputs.split()
-            inputs = inputs.split()
-            for o in outputs:
-                try:
-                    result[o].extend(inputs)
-                except KeyError:
-                    result[o] = inputs
-    return result
-
-def check_rule(ui, repo, modified, basedir, output, inputs):
-    """Verify that the output is newer than any of the inputs.
-    Return (status, stamp), where status is True if the update succeeded,
-    and stamp is the newest time stamp assigned  to any file (might be in
-    the future).
-
-    If basedir is nonempty, it gives a directory in which the tree is to
-    be checked.
-    """
-    f_output = repo.wjoin(os.path.join(basedir, output))
-    try:
-        o_time = os.stat(f_output).st_mtime
-    except OSError:
-        ui.warn("Generated file %s does not exist\n" % output)
-        return False, 0
-    youngest = 0   # youngest dependency
-    backdate = None
-    backdate_source = None
-    for i in inputs:
-        f_i = repo.wjoin(os.path.join(basedir, i))
-        try:
-            i_time = os.stat(f_i).st_mtime
-        except OSError:
-            ui.warn(".hgtouch input file %s does not exist\n" % i)
-            return False, 0
-        if i in modified:
-            # input is modified. Need to backdate at least to i_time
-            if backdate is None or backdate > i_time:
-                backdate = i_time
-                backdate_source = i
-            continue
-        youngest = max(i_time, youngest)
-    if backdate is not None:
-        ui.warn("Input %s for file %s locally modified\n" % (backdate_source, output))
-        # set to 1s before oldest modified input
-        backdate -= 1
-        os.utime(f_output, (backdate, backdate))
-        return False, 0
-    if youngest >= o_time:
-        ui.note("Touching %s\n" % output)
-        youngest += 1
-        os.utime(f_output, (youngest, youngest))
-        return True, youngest
-    else:
-        # Nothing to update
-        return True, 0
-
-def do_touch(ui, repo, basedir):
-    if basedir:
-        if not os.path.isdir(repo.wjoin(basedir)):
-            ui.warn("Abort: basedir %r does not exist\n" % basedir)
-            return
-        modified = []
-    else:
-        modified = repo.status()[0]
-    dependencies = parse_config(repo)
-    success = True
-    tstamp = 0       # newest time stamp assigned
-    # try processing all rules in topological order
-    hold_back = {}
-    while dependencies:
-        output, inputs = dependencies.popitem()
-        # check whether any of the inputs is generated
-        for i in inputs:
-            if i in dependencies:
-                hold_back[output] = inputs
-                continue
-        _success, _tstamp = check_rule(ui, repo, modified, basedir, output, inputs)
-        success = success and _success
-        tstamp = max(tstamp, _tstamp)
-        # put back held back rules
-        dependencies.update(hold_back)
-        hold_back = {}
-    now = time.time()
-    if tstamp > now:
-        # wait until real time has passed the newest time stamp, to
-        # avoid having files dated in the future
-        time.sleep(tstamp-now)
-    if hold_back:
-        ui.warn("Cyclic dependency involving %s\n" % (' '.join(hold_back.keys())))
-        return False
-    return success
-
-def touch(ui, repo, basedir):
-    "touch generated files that are older than their sources after an update."
-    do_touch(ui, repo, basedir)
-
-cmdtable = {
-    "touch": (touch,
-              [('b', 'basedir', '', 'base dir of the tree to apply touching')],
-              "hg touch [-b BASEDIR]")
-}