# @+leo-ver=5-thin
# @+node:ekr.20150323150718.1: * @file leoAtFile.py
"""Classes to read and write @file nodes."""

# @+<< leoAtFile imports & annotations >>
# @+node:ekr.20041005105605.2: ** << leoAtFile imports & annotations >>
from __future__ import annotations
from collections.abc import Callable
import difflib
import io
import os
import re
import sys
import tabnanny
import time
import tokenize
from typing import Any, Optional, TYPE_CHECKING
from leo.core import leoGlobals as g
from leo.core import leoNodes

if TYPE_CHECKING:  # pragma: no cover
    from leo.core.leoCommands import Commands as Cmdr
    from leo.core.leoGui import LeoKeyEvent
    from leo.core.leoNodes import Position, VNode

    Args = Any
    Value = Any
# @-<< leoAtFile imports & annotations >>


# @+others
# @+node:ekr.20150509194251.1: ** cmd (decorator)
def cmd(name: str) -> Callable:  # pragma: no cover
    """Command decorator for the AtFileCommands class."""
    return g.new_cmd_decorator(name, ['c', 'atFileCommands'])


# @+node:ekr.20160514120655.1: ** class AtFile
class AtFile:
    """A class implementing the atFile subcommander."""

    # @+<< AtFile: define __slots__ >>
    # @+node:ekr.20250403114721.1: *3* << AtFile: define __slots__ >>
    __slots__ = (
        # Ivars.
        'c',
        'fileCommands',
        # Basic status vars.
        'errors',
        'language',
        'root',
        # Dialogs.
        'canCancelFlag',
        'cancelFlag',
        'yesToAll',
        # Reading.
        'readVersion',
        'read_i',
        'read_lines',
        'importRootSeen',
        'startSentinelComment',
        'endSentinelComment',
        # @verbatim
        # @shadow and @clean files.
        'bodies_dict',
        'changed_roots',
        'private_s',
        'public_s',
        # Writing.
        'indent',
        'sentinels',
        'section_delim1',
        'section_delim2',
        'outputFile',
        'outputList',
        'targetFileName',
        'unchangedFiles',  # For messages.
        # User settings.
        'at_auto_encoding',
        'encoding',
        'explicitLineEnding',
        'force_newlines_in_at_nosent_bodies',
        'output_newline',
        'page_width',
        'tab_width',
        # User switches.
        'beautifyOnWrite',
        'checkPythonCodeOnWrite',
        'runFlake8OnWrite',
        'runPyFlakesOnWrite',
        'runPylintOnWrite',
        # Testing hacks.
        'at_shadow_test_hack',  # Injected by TestShadow.makePrivateLines.
    )
    # @-<< AtFile: define __slots__ >>
    # @+<< AtFile: define constants >>
    # @+node:ekr.20131224053735.16380: *3* << AtFile: define constants >>
    # fmt: off

    # directives...
    noDirective     =  1  # not an at-directive.
    allDirective    =  2  # @all (4.2)
    docDirective    =  3  # @doc.
    atDirective     =  4  # @<space or newline>
    codeDirective   =  5  # @code
    cDirective      =  6  # @c<space or newline>
    othersDirective =  7  # @others
    miscDirective   =  8  # All other directives
    startVerbatim   =  9  # @verbatim  Not a real directive. Used to issue warnings.

    # fmt: on
    # @-<< AtFile: define constants >>
    # @+others
    # @+node:ekr.20041005105605.7: *3* at.Birth & init
    # @+node:ekr.20041005105605.8: *4* at.ctor & helpers
    # Note: g.getScript also call the at.__init__ and at.finishCreate().

    def __init__(self, c: Cmdr) -> None:
        """ctor for atFile class."""
        # Ivars.
        self.c: Cmdr = c
        self.fileCommands = c.fileCommands
        # Basic status vars.
        self.errors = 0
        self.language: str = None
        self.root: Position = None
        # Dialogs.
        self.canCancelFlag = False
        self.cancelFlag = False
        self.yesToAll = False
        # Reading.
        self.changed_roots: list[Position] = []  # Global.
        self.bodies_dict: dict[VNode, str] = {}  # Local to at.readOneAtCleanNode.
        self.importRootSeen = False
        self.readVersion = ''
        self.read_i = 0
        self.read_lines: list[str] = []
        self.startSentinelComment = ""
        self.endSentinelComment = ""
        # Writing.
        self.indent = 0  # write indentation, in blanks.
        self.outputFile: io.StringIO = None
        self.outputList: list[str] = []
        self.sentinels = False
        self.section_delim1 = '<<'
        self.section_delim2 = '>>'
        self.targetFileName: str = ''
        self.unchangedFiles = 0
        # User settings.
        self.at_auto_encoding = 'utf-8'
        self.encoding = 'utf-8'
        self.explicitLineEnding: bool = None
        self.force_newlines_in_at_nosent_bodies = False
        self.output_newline = g.getOutputNewline(c=c)
        self.page_width: int = None
        self.tab_width: int = c.tab_width or -4
        # User switches: set in reloadSettings.
        self.beautifyOnWrite = False
        self.checkPythonCodeOnWrite = False
        self.runFlake8OnWrite = False
        self.runPyFlakesOnWrite = False
        self.runPylintOnWrite = False
        # Initialize all user switches.
        self.reloadSettings()

    # @+node:ekr.20171113152939.1: *5* at.reloadSettings
    def reloadSettings(self) -> None:
        """AtFile.reloadSettings"""
        c = self.c
        self.beautifyOnWrite = c.config.getBool('beautify-python-code-on-write', default=False)
        self.checkPythonCodeOnWrite = c.config.getBool('check-python-code-on-write', default=True)
        self.runFlake8OnWrite = c.config.getBool('run-flake8-on-write', default=False)
        self.runPyFlakesOnWrite = c.config.getBool('run-pyflakes-on-write', default=False)
        self.runPylintOnWrite = c.config.getBool('run-pylint-on-write', default=False)

    # @+node:ekr.20250403154610.1: *4* at.initAllIvars
    def initAllIvars(self, root: Position) -> None:
        """Init all ivars to reasonable defaults."""
        at, c = self, self.c
        assert root, g.callers()
        # Basic status vars.
        at.errors = 0
        at.language = c.target_language or 'python'
        at.root = root
        # Reading
        at.importRootSeen = False
        at.readVersion = ''
        at.read_i = 0
        at.read_lines = []
        at.startSentinelComment = ""
        at.endSentinelComment = ""
        # Writing.
        at.indent = 0  # Output indentation, in blanks.
        at.outputFile = None
        at.outputList = []
        at.sentinels = False
        at.section_delim1 = '<<'
        at.section_delim2 = '>>'
        at.targetFileName = None
        # at.unchangedFiles = 0  # Only at.writeAll should init this ivar.
        # User settings.
        at.at_auto_encoding = c.config.default_at_auto_file_encoding or 'utf-8'
        at.encoding = c.config.default_derived_file_encoding or 'utf-8'
        at.explicitLineEnding = None
        at.force_newlines_in_at_nosent_bodies = False
        at.output_newline = g.getOutputNewline(c=c)
        at.page_width = c.page_width or 132
        at.tab_width = c.tab_width or -4

    # @+node:ekr.20041005105605.13: *4* at.initReadIvars
    def initReadIvars(self, root: Position, fileName: str) -> None:
        """Initialize all read ivars."""

        # Set all ivars to reasonable defaults.
        self.initAllIvars(root)

    # @+node:ekr.20041005105605.15: *4* at.initWriteIvars
    def initWriteIvars(self, root: Position) -> Optional[str]:
        """
        Compute default values of all write-related ivars.
        Return the finalized name of the output file.
        """
        at, c = self, self.c
        if not c or not c.config:
            return None  # pragma: no cover

        make_dirs = c.config.getBool('create-nonexistent-directories', default=False)
        assert at.checkPythonCodeOnWrite is not None

        # Set all ivars to reasonable defaults.
        at.initAllIvars(root)

        # Set other ivars.
        at.sentinels = True
        at.force_newlines_in_at_nosent_bodies = c.config.getBool(
            'force-newlines-in-at-nosent-bodies'
        )
        # For at.putBody only.

        at.encoding = c.getEncoding(root)
        lineending = c.getLineEnding(root)
        at.explicitLineEnding = bool(lineending)
        if g.unitTesting:
            at.output_newline = '\n'
        else:
            at.output_newline = lineending or g.getOutputNewline(c=c)
        at.language = c.getLanguage(root)
        at.outputList = []  # For stream output.
        at.page_width = c.getPageWidth(root)
        at.tab_width = c.getTabWidth(root)
        # More complex inits.
        at.initSentinelComments(root)

        # Clean root.v.
        if not at.errors and at.root:
            at.root.v._p_changed = True

        # #1907: Compute the file name and create directories as needed.
        targetFileName = g.os_path_realpath(c.fullPath(root))
        at.targetFileName = targetFileName  # For at.writeError only.

        # targetFileName can be empty for unit tests & @command nodes.
        if not targetFileName:  # pragma: no cover
            targetFileName = root.h if g.unitTesting else None
            at.targetFileName = targetFileName  # For at.writeError only.
            return targetFileName

        # #2276: scan for section delims
        at.scanRootForSectionDelims(root)

        # Do nothing more if the file already exists.
        if os.path.exists(targetFileName):
            return targetFileName

        # Create directories if enabled.
        root_dir = g.os_path_dirname(targetFileName)
        if make_dirs and root_dir:  # pragma: no cover
            ok = g.makeAllNonExistentDirectories(root_dir)
            if not ok:
                g.error(f"Error creating directories: {root_dir}")
                return None

        # Return the target file name, regardless of future problems.
        return targetFileName

    # @+node:ekr.20250405052328.1: *4* at.initSentinelComments
    def initSentinelComments(self, root: Position) -> None:
        """Init at.startSentinelComment and at.endSentinelComment."""
        at, c = self, self.c
        delim1, delim2, delim3 = c.getDelims(root)

        # Use single-line comments if we have a choice.
        # delim1,delim2,delim3 now correspond to line,start,end
        if delim1:
            at.startSentinelComment = delim1
            at.endSentinelComment = ""  # Must not be None.
        elif delim2 and delim3:
            at.startSentinelComment = delim2
            at.endSentinelComment = delim3
        else:  # pragma: no cover
            #
            # Emergency!
            #
            # Issue an error only if at.language has been set.
            # This suppresses a message from the markdown importer.
            if not g.unitTesting and at.language:
                g.trace(repr(at.language), g.callers())
                g.es_print(f"unknown language: {at.language}")
                g.es_print('using Python comment delimiters')
            at.startSentinelComment = "#"  # This should never happen!
            at.endSentinelComment = ""

    # @+node:ekr.20041005105605.17: *3* at.Reading
    # @+node:ekr.20041005105605.18: *4* at.Reading (top level)
    # @+node:ekr.20070919133659: *5* at.checkExternalFile
    @cmd('check-external-file')
    def checkExternalFile(self, event: LeoKeyEvent = None) -> None:  # pragma: no cover
        """Make sure an external file written by Leo may be read properly."""
        c, p = self.c, self.c.p
        if not p.isAtFileNode() and not p.isAtThinFileNode():
            g.red('Please select an @thin or @file node')
            return
        fn = c.fullPath(p)  # #1910.
        if not g.os_path_exists(fn):
            g.red(f"file not found: {fn}")
            return
        s, e = g.readFileIntoString(fn)
        if s is None:
            g.red(f"empty file: {fn}")
            return
        #
        # Create a dummy, unconnected, VNode as the root.
        root_v = leoNodes.VNode(context=c)
        root = leoNodes.Position(root_v)
        FastAtRead(c, gnx2vnode={}).read_into_root(s, fn, root)

    # @+node:ekr.20250724123631.1: *5* at.openAtLeoFile
    @cmd('open-at-leo-file')
    def openAtLeoFile(self, event: LeoKeyEvent = None) -> None:  # pragma: no cover
        """
        Open the outline given by the @leo node at c.p.
        If the outline has already been loaded, switch to its tab.

        Scripts should use c.makeLinkLeoFiles helper to make @leo files.
        """
        c, p = self.c, self.c.p
        if not p.isAtLeoNode():
            g.red('Please select an @leo node')
            return
        path = c.fullPath(p)
        if g.os_path_exists(path):
            g.openWithFileName(path, old_c=c)
        else:
            g.red(f"file not found: {path}")

    # @+node:ekr.20041005105605.19: *5* at.openFileForReading & helper
    def openFileForReading(self, fromString: str = None) -> tuple[Optional[str], Optional[str]]:
        """
        Open the file given by at.root.
        This will be the private file for @shadow nodes.
        """
        at, c = self, self.c
        is_at_shadow = self.root.isAtShadowFileNode()
        if fromString:  # pragma: no cover
            if is_at_shadow:  # pragma: no cover
                at.error('can not call at.read from string for @shadow files')
                return None, None
            at.initReadLine(fromString)
            return None, None
        #
        # Not from a string. Carefully read the file.
        # Returns full path, including file name.
        fn = g.fullPath(c, at.root)
        # Remember the full path to this node.
        at.setPathUa(at.root, fn)
        if is_at_shadow:  # pragma: no cover
            fn = at.openAtShadowFileForReading(fn)
            if not fn:
                return None, None
        assert fn
        try:
            # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
            s = at.readFileToUnicode(fn)
            # #1466.
            if s is None:  # pragma: no cover
                # The error has been given.
                return None, None
            at.warnOnReadOnlyFile(fn)
        except Exception:  # pragma: no cover
            at.error(f"unexpected exception opening: '@file {fn}'")
            fn, s = None, None
        return fn, s

    # @+node:ekr.20150204165040.4: *6* at.openAtShadowFileForReading
    def openAtShadowFileForReading(self, fn: str) -> Optional[str]:  # pragma: no cover
        """Open an @shadow for reading and return shadow_fn."""
        at = self
        x = at.c.shadowController
        # readOneAtShadowNode should already have checked these.
        shadow_fn = x.shadowPathName(fn)
        shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
        if not shadow_exists:
            g.trace('can not happen: no private file', shadow_fn, g.callers())
            at.error(f"can not happen: private file does not exist: {shadow_fn}")
            return None
        # This method is the gateway to the shadow algorithm.
        x.updatePublicAndPrivateFiles(at.root, fn, shadow_fn)
        return shadow_fn

    # @+node:ekr.20041005105605.21: *5* at.read & helpers
    def read(self, root: Position, fromString: str = None) -> bool:
        """Read an @thin or @file tree."""
        at, c = self, self.c
        fileName = c.fullPath(root)
        if not fileName:  # pragma: no cover
            at.error("Missing file name. Restoring @file tree from .leo file.")
            return False

        # #760531: always mark the root as read, even if there was an error.
        # #889175: Remember the full fileName.
        at.rememberReadPath(c.fullPath(root), root)
        at.initReadIvars(root, fileName)
        if at.errors:
            return False  # pragma: no cover

        # Open the file.
        fileName, file_s = at.openFileForReading(fromString=fromString)
        if file_s is None:  # #1798:
            return False  # pragma: no cover

        # Set the time stamp.
        if fileName:
            c.setFileTimeStamp(fileName)
        elif not fileName and not fromString and not file_s:  # pragma: no cover
            return False

        # Read the file!
        root.clearVisitedInTree()
        gnx2vnode = c.fileCommands.gnxDict
        contents = fromString or file_s
        FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
        root.clearDirty()
        g.doHook('after-reading-external-file', c=c, p=root)
        return True

    # @+node:ekr.20071105164407: *6* at.deleteUnvisitedNodes
    def deleteUnvisitedNodes(self, root: Position) -> None:  # pragma: no cover
        """
        Delete unvisited nodes in root's subtree, not including root.

        Before Leo 5.6: Move unvisited node to be children of the 'Resurrected
        Nodes'.
        """
        at, c = self, self.c
        # Find the unvisited nodes.
        aList = [z for z in root.subtree() if not z.isVisited()]
        if aList:
            at.c.deletePositionsInList(aList)
            c.redraw()

    # @+node:ekr.20041005105605.26: *5* at.readAll & helpers
    def readAll(self, root: Position) -> None:
        """Scan positions, looking for @<file> nodes to read."""
        at, c = self, self.c
        at.changed_roots = []
        old_changed = c.changed
        t1 = time.time()
        c.init_error_dialogs()
        files = at.findFilesToRead(root, all=True)
        efc = g.app.externalFilesController
        for p in files:
            at.readFileAtPosition(p)
            # An @<file> was read: In case this was a tab opened earlier, for which efc had entries,
            # we need to update the external files controller's timestamp for those external files
            # instead of leaving the _time_d empty, which would have defaulted to set_time's default.
            if efc:
                efc.set_time(c.fullPath(p))  # #4426 Same effect as leaving efc's _time_d empty.
        for p in files:
            p.v.clearDirty()
        if not g.unitTesting and files:  # pragma: no cover
            t2 = time.time()
            g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")

        # Carefully set c.changed.
        c.changed = old_changed or bool(at.changed_roots)
        update_p = at.clone_all_changed_vnodes()
        if update_p:
            # Select update_p.  See fc.setPositionsFromVnodes.
            c.db['current_position'] = ','.join([str(z) for z in update_p.archivedPosition()])
            update_p.expand()

        at.changed_roots = []

        # Last.
        c.raise_error_dialogs()

    # @+node:ekr.20250711132317.1: *6* at.clone_all_changed_vnodes
    def clone_all_changed_vnodes(self) -> Position:
        """
        Make clones of all changed VNodes.

        Called from at.readAll, at.readAllSelected and c.refreshFromDisk.
        Callers are responsible for setting c.p and redrawing.
        """
        at, c, u = self, self.c, self.c.undoer

        if g.unitTesting:
            return None
        if not at.changed_roots:
            return None
        if not c.config.getBool('report-changed-at-clean-nodes', default=False):
            return None

        # Undoably create the top-level node.
        undoData = u.beforeInsertNode(c.p)
        update_p = c.lastTopLevel().insertAfter()
        update_p.h = 'Updated @clean/@auto nodes'

        # Clone nodes as children of the found node.
        for root in at.changed_roots:
            parent = update_p.insertAsLastChild()
            parent.h = f"Updated from: {g.shortFileName(c.fullPath(root))}"
            parent_body: list[str] = []
            # Clone all dirty nodes.
            root.v.setDirty()
            for p in root.self_and_subtree():
                v = p.v
                if not v.isDirty():
                    continue
                clone = p.clone()
                clone.moveToLastChildOf(parent)
                # Insert the diff into the parent's body.
                a = g.splitLines(at.bodies_dict.get(v, ''))
                b = g.splitLines(p.b)
                parent_body.append(f"{p.h}\n")
                parent_body.extend(difflib.unified_diff(a, b))
                parent_body.append('\n')
            # Put the diff.
            parent.b = ''.join(parent_body)

        # Defensive programming.
        if c.checkOutline() > 0:
            return None

        # Sort the clones in place, without undo.
        update_p.v.children.sort(key=lambda v: v.h.lower())
        u.afterInsertNode(update_p, 'Clone Updated Nodes', undoData)
        c.contractAllHeadlinesCommand()
        update_p.expand()
        return update_p

    # @+node:ekr.20190108054317.1: *6* at.findFilesToRead
    def findFilesToRead(self, root: Position, all: bool) -> list[Position]:  # pragma: no cover
        c = self.c
        p = root.copy()
        scanned_nodes: set[tuple[str, str]] = set()
        files: list[Position] = []
        after = None if all else p.nodeAfterTree()
        while p and p != after:
            data = (p.gnx, c.fullPath(p))
            # skip clones referring to exactly the same paths.
            if data in scanned_nodes:
                p.moveToNodeAfterTree()
                continue
            scanned_nodes.add(data)
            if not p.h.startswith('@'):
                p.moveToThreadNext()
            elif p.isAtIgnoreNode():
                if p.isAnyAtFileNode():
                    c.ignored_at_file_nodes.append(p.h)
                p.moveToNodeAfterTree()
            elif (
                p.isAtThinFileNode()
                or p.isAtAutoNode()
                or p.isAtEditNode()
                or p.isAtShadowFileNode()
                or p.isAtFileNode()
                or p.isAtCleanNode()
                or p.isAtJupytextNode()
            ):
                files.append(p.copy())
                p.moveToNodeAfterTree()
            elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
                # Note (see #1081): @asis and @nosent can *not* be updated automatically.
                # Doing so using refresh-from-disk will delete all child nodes.
                p.moveToNodeAfterTree()
            else:
                p.moveToThreadNext()
        return files

    # @+node:ekr.20190108054803.1: *6* at.readFileAtPosition
    def readFileAtPosition(self, p: Position) -> None:  # pragma: no cover
        """
        Read the @<file> node at p."""
        at, c = self, self.c

        if p.isAtAsisFileNode():
            at.readOneAtAsisNode(p)  # Changed.
        elif p.isAtAutoNode() or p.isAtAutoRstNode():
            old_gnx = p.v.gnx
            p = at.readOneAtAutoNode(p)  # Might change p!
            # Give a weird error.
            if p.v.gnx != old_gnx:
                g.es_print(f"reading @auto node changed the gnx for `{p.h}`")
                g.es_print(f"from `{old_gnx}` to: `{p.v.gnx}`")
                c.selectPosition(p)
        elif p.isAtCleanNode():
            at.readOneAtCleanNode(p)
        elif p.isAtEditNode():
            at.readOneAtEditNode(p)
        elif p.isAtFileNode() or p.isAtThinFileNode():
            at.read(p)
        elif p.isAtJupytextNode():
            at.readOneAtJupytextNode(p)
        elif p.isAtNoSentFileNode():
            at.rememberReadPath(c.fullPath(p), p)
        elif p.isAtShadowFileNode():
            fileName = p.anyAtFileNodeName()
            at.readOneAtShadowNode(fileName, p)

    # @+node:ekr.20220121052056.1: *5* at.readAllSelected
    def readAllSelected(self, root: Position) -> None:  # pragma: no cover
        """Read all @<file> nodes in root's tree."""
        at, c = self, self.c
        at.changed_roots = []
        old_changed = c.changed
        t1 = time.time()
        c.init_error_dialogs()
        files = at.findFilesToRead(root, all=False)
        for p in files:
            at.readFileAtPosition(p)
        for p in files:
            p.v.clearDirty()
        if not g.unitTesting:  # pragma: no cover
            if files:
                t2 = time.time()
                g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
            else:
                g.es("no @<file> nodes in the selected tree")

        # Carefully set c.changed.
        c.changed = old_changed or bool(at.changed_roots)
        update_p = at.clone_all_changed_vnodes()
        if update_p:
            # Select update_p.  See fc.setPositionsFromVnodes.
            c.db['current_position'] = ','.join([str(z) for z in update_p.archivedPosition()])
            update_p.expand()

        at.changed_roots = []

        # Last.
        c.raise_error_dialogs()

    # @+node:ekr.20080801071227.7: *5* at.readAtShadowNodes
    def readAtShadowNodes(self, p: Position) -> None:  # pragma: no cover
        """Read all @shadow nodes in the p's tree."""
        at = self
        after = p.nodeAfterTree()
        p = p.copy()  # Don't change p in the caller.
        while p and p != after:  # Don't use iterator.
            if p.isAtShadowFileNode():
                fileName = p.atShadowFileNodeName()
                at.readOneAtShadowNode(fileName, p)
                p.moveToNodeAfterTree()
            else:
                p.moveToThreadNext()

    # @+node:ekr.20190201104956.1: *5* at.readOneAtAsisNode
    def readOneAtAsisNode(self, p: Position) -> None:  # pragma: no cover
        """Read one @asis node. Used only by refresh-from-disk"""
        at, c = self, self.c
        fn = c.fullPath(p)
        junk, ext = g.os_path_splitext(fn)
        # Remember the full fileName.
        at.rememberReadPath(fn, p)
        s, e = g.readFileIntoString(fn, kind='@edit')
        if s is None:
            return
        encoding = 'utf-8' if e is None else e
        # Delete all children.
        while p.hasChildren():
            p.firstChild().doDelete()
        old_body = p.b
        p.b = g.toUnicode(s, encoding=encoding, reportErrors=True)
        if not c.isChanged() and p.b != old_body:
            c.setChanged()
        g.doHook('after-reading-external-file', c=c, p=p)

    # @+node:ekr.20070909100252: *5* at.readOneAtAutoNode
    def readOneAtAutoNode(self, p: Position) -> Position:  # pragma: no cover
        """Read an @auto file into p. Return the *new* position."""
        at, c, ic = self, self.c, self.c.importCommands
        fileName = c.fullPath(p)
        if not g.os_path_exists(fileName):
            g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
            return p
        # Remember that we have seen the @auto node.
        # #889175: Remember the full fileName.
        at.rememberReadPath(fileName, p)
        old_p = p.copy()
        try:
            at.initReadIvars(p, fileName)
            p.v.b = ''  # Required for @auto API checks.
            p.v._deleteAllChildren()
            p = ic.createOutline(parent=p.copy(), treeType='@auto')
            # Do *not* call c.selectPosition(p) here.
            # That would improperly expand nodes.
        except Exception:
            p = old_p
            ic.errors += 1
            g.es_print('Unexpected exception importing', fileName)
            g.es_exception()
        if ic.errors:
            g.error(f"errors inhibited read @auto {fileName}")
        elif c.persistenceController:
            c.persistenceController.update_after_read_foreign_file(p)
        # Finish.
        if ic.errors or not g.os_path_exists(fileName):
            p.clearDirty()
        else:
            g.doHook('after-auto', c=c, p=p)
            g.doHook('after-reading-external-file', c=c, p=p)
        return p  # For #451: return p.

    # @+node:ekr.20150204165040.5: *5* at.readOneAtCleanNode & helpers
    def readOneAtCleanNode(self, root: Position, *, new_contents: str = None) -> None:
        """Update the @clean/@nosent node at root."""
        at, c, x = self, self.c, self.c.shadowController

        if new_contents:
            fileName = root.h  # Required.
        else:
            fileName = c.fullPath(root)
            if not g.os_path_exists(fileName):
                g.es_print(f"not found: {fileName}", color='red', nodeLink=root.get_UNL())
                return
            # Suppresses file-changed dialog.
            at.rememberReadPath(fileName, root)

        # #4385: Do nothing if the file has not changed.
        try:
            old_mod_time = root.v.u['_mod_time']  # #4385 The file's *last-seen* mod time.
        except Exception:
            old_mod_time = None
        new_mod_time = g.os_path_getmtime(fileName)  # The file's *present* mod time.

        # Make sure it's newer: Don't update if the outline and file are in synch.
        if old_mod_time and old_mod_time >= new_mod_time:
            return

        # #4385: Init the per-file data.
        at.initReadIvars(root, fileName)
        at.bodies_dict = {}

        # #4385: *Clear* the mod time until we write the file.
        if '_mod_time' in root.v.u:
            del root.v.u['_mod_time']

        # Until the @clean's content is modified and written: set to file's *present* mod time.
        # This and writeOneAtCleanNode are the *only* two places that sets the `_mod_time` uA.
        root.v.u['_mod_time'] = new_mod_time  # #4427

        # #4385: Remember all old bodies.
        for p in root.self_and_subtree():
            at.bodies_dict[p.v] = p.b

        # Calculate data.
        new_public_lines = (
            g.splitLines(new_contents) if new_contents else at.read_at_clean_lines(fileName)
        )
        old_private_lines = self.write_at_clean_sentinels(root)
        marker = x.markerFromFileLines(old_private_lines, fileName)
        old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
        if old_public_lines:
            new_private_lines = x.propagate_changed_lines(
                new_public_lines, old_private_lines, marker, p=root
            )
        else:
            new_private_lines = []
            root.b = ''.join(new_public_lines)
            return
        if new_private_lines == old_private_lines:
            return
        root.clearVisitedInTree()
        gnx2vnode = at.fileCommands.gnxDict
        contents = ''.join(new_private_lines)
        FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
        g.doHook('after-reading-external-file', c=c, p=root)

        # Calculate all changed vnodes.
        changed_vnodes: list[VNode] = []
        for p in root.self_and_subtree():
            v = p.v
            if at.bodies_dict.get(v) != p.b:
                changed_vnodes.append(v)
                v.setDirty()

        # Handle all changed vnodes.
        if changed_vnodes:
            c.setChanged(force=True)
            root.v.setDirty()
            at.changed_roots.append(root.copy())

    # @+node:ekr.20150204165040.8: *6* at.read_at_clean_lines
    def read_at_clean_lines(self, fn: str) -> list[str]:  # pragma: no cover
        """Return all lines of the @clean/@nosent file at fn."""
        at = self
        # Use the standard helper. Better error reporting.
        # Important: uses 'rb' to open the file.
        s: str
        s_bytes: bytes = at.openFileHelper(fn)
        # #1798.
        if not s_bytes:
            s = ''
        else:
            s = g.toUnicode(s_bytes, encoding=at.encoding)
            s = s.replace('\r\n', '\n')  # Suppress meaningless "node changed" messages.
        return g.splitLines(s)

    # @+node:ekr.20150204165040.9: *6* at.write_at_clean_sentinels
    def write_at_clean_sentinels(self, root: Position) -> list[str]:  # pragma: no cover
        """
        Return all lines of the @clean tree as if it were
        written as an @file node.
        """
        at = self
        result = at.atFileToString(root, sentinels=True)
        s = g.toUnicode(result, encoding=at.encoding)
        return g.splitLines(s)

    # @+node:ekr.20090225080846.3: *5* at.readOneAtEditNode
    def readOneAtEditNode(self, p: Position) -> None:  # pragma: no cover
        at = self
        c = at.c
        ic = c.importCommands
        fn = c.fullPath(p)
        junk, ext = g.os_path_splitext(fn)
        # Fix bug 889175: Remember the full fileName.
        at.rememberReadPath(fn, p)
        s, e = g.readFileIntoString(fn, kind='@edit')
        if s is None:
            return
        encoding = 'utf-8' if e is None else e
        # Delete all children.
        while p.hasChildren():
            p.firstChild().doDelete()
        head = ''
        ext = ext.lower()
        if ext in ('.html', '.htm'):
            head = '@language html\n'
        elif ext in ('.txt', '.text'):
            head = '@nocolor\n'
        else:
            language = ic.languageForExtension(ext)
            if language and language != 'unknown_language':
                head = f"@language {language}\n"
            else:
                head = '@nocolor\n'
        p.b = head + g.toUnicode(s, encoding=encoding, reportErrors=True)
        g.doHook('after-edit', p=p)
        g.doHook('after-reading-external-file', c=c, p=p)

    # @+node:ekr.20241023135739.1: *5* at.readOneAtJupytextNode
    def readOneAtJupytextNode(self, root: Position) -> None:  # pragma: no cover
        """
        p must be an @jupytext node.
        - Convert the .ipynb file to a string s.
        - Update p.b's tree using s.

        This code is adapted from at.readOneAtCleanNode.
        """
        at, c, x = self, self.c, self.c.shadowController
        contents, fileName = g.app.jupytextManager.read(c, root)
        if not contents:
            return
        if not os.path.exists(fileName):
            g.internalError(f"does not exist: {fileName!r}")
            return

        # Init.
        at.rememberReadPath(fileName, root)
        at.initReadIvars(root, fileName)

        # Always use python comments.
        at.startSentinelComment = '#'
        at.endSentinelComment = ""  # Must not be None.
        delims = '#', '', ''
        marker = x.Marker(delims)

        # The body of the update algorithm.
        new_public_lines = g.splitLines(contents)
        old_private_lines = self.write_at_clean_sentinels(root)
        old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
        new_private_lines = x.propagate_changed_lines(
            new_public_lines, old_private_lines, marker, p=root
        )
        if old_public_lines:  # Don't even *think* about removing this test.
            new_private_lines = x.propagate_changed_lines(
                new_public_lines, old_private_lines, marker, p=root
            )
        else:
            # This is not an error, it is
            # the first time the @jupytext has been read.
            new_private_lines = []
            root.b = ''.join(new_public_lines)
            g.app.jupytextManager.create_outline(c, root)
            g.doHook('after-reading-external-file', c=c, p=root)
            return
        if new_private_lines == old_private_lines:
            return
        root.clearVisitedInTree()
        gnx2vnode = at.fileCommands.gnxDict
        new_contents = '@language python\n' + ''.join(new_private_lines)
        FastAtRead(c, gnx2vnode).read_into_root(new_contents, fileName, root)
        g.doHook('after-reading-external-file', c=c, p=root)

    # @+node:ekr.20080711093251.7: *5* at.readOneAtShadowNode & helper
    def readOneAtShadowNode(self, fn: str, p: Position) -> None:  # pragma: no cover
        at, c = self, self.c
        x = c.shadowController
        if not fn == p.atShadowFileNodeName():
            at.error(f"can not happen: fn: {fn} != atShadowNodeName: {p.atShadowFileNodeName()}")
            return
        fn = c.fullPath(p)
        # #889175: Remember the full fileName.
        at.rememberReadPath(fn, p)
        shadow_fn = x.shadowPathName(fn)
        shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
        # Delete all children.
        while p.hasChildren():
            p.firstChild().doDelete()
        if shadow_exists:
            at.read(p)
        else:
            ok = at.importAtShadowNode(p)
            if ok:
                # Create the private file automatically.
                at.writeOneAtShadowNode(p)
                g.doHook('after-reading-external-file', c=c, p=p)

    # @+node:ekr.20080712080505.1: *6* at.importAtShadowNode
    def importAtShadowNode(self, p: Position) -> bool:  # pragma: no cover
        c, ic = self.c, self.c.importCommands
        fn = c.fullPath(p)
        if not g.os_path_exists(fn):
            g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
            return False
        # Delete all the child nodes.
        while p.hasChildren():
            p.firstChild().doDelete()
        # Import the outline, exactly as @auto does.
        ic.createOutline(parent=p.copy(), treeType='@auto')
        if ic.errors:
            g.error('errors inhibited read @shadow', fn)
        if ic.errors or not g.os_path_exists(fn):
            p.clearDirty()
        return ic.errors == 0

    # @+node:ekr.20180622110112.1: *4* at.fast_read_into_root
    def fast_read_into_root(
        self,
        c: Cmdr,
        contents: str,
        gnx2vnode: dict[str, VNode],
        path: str,
        root: Position,
    ) -> bool:  # pragma: no cover
        """A convenience wrapper for FastAtRead.read_into_root()"""
        return FastAtRead(c, gnx2vnode).read_into_root(contents, path, root)

    # @+node:ekr.20041005105605.116: *4* at.Reading utils...
    # @+node:ekr.20041005105605.119: *5* at.createImportedNode
    def createImportedNode(self, root: Position, headline: str) -> Position:  # pragma: no cover
        at = self
        if at.importRootSeen:
            p = root.insertAsLastChild()
            p.initHeadString(headline)
        else:
            # Put the text into the already-existing root node.
            p = root
            at.importRootSeen = True
        p.v.setVisited()  # Suppress warning about unvisited node.
        return p

    # @+node:ekr.20130911110233.11286: *5* at.initReadLine
    def initReadLine(self, s: str) -> None:
        """Init the ivars so that at.readLine will read all of s."""
        at = self
        at.read_i = 0
        at.read_lines = g.splitLines(s)

    # @+node:ekr.20041005105605.120: *5* at.parseLeoSentinel
    def parseLeoSentinel(self, s: str) -> tuple[bool, bool, str, str, bool]:
        """
        Parse the sentinel line s.
        If the sentinel is valid, set at.encoding, at.readVersion
        """
        at, c = self, self.c
        # Set defaults.
        encoding = c.config.default_derived_file_encoding
        readVersion = None
        new_df, start, end, isThin = False, '', '', False
        # Example: \*@+leo-ver=5-thin-encoding=utf-8,.*/
        pattern = re.compile(r'(.+)@\+leo(-ver=([0123456789]+))?(-thin)?(-encoding=(.*)(\.))?(.*)')
        # The old code weirdly allowed '.' in version numbers.
        # group 1: opening delim
        # group 2: -ver=
        # group 3: version number
        # group(4): -thin
        # group(5): -encoding=utf-8,.
        # group(6): utf-8,
        # group(7): .
        # group(8): closing delim.
        m = pattern.match(s)
        valid = bool(m)
        if valid:
            start = m.group(1)  # start delim
            valid = bool(start)
        if valid:
            new_df = bool(m.group(2))  # -ver=
            if new_df:
                # Set the version number.
                if m.group(3):
                    readVersion = m.group(3)
                else:
                    valid = False  # pragma: no cover
        if valid:
            # set isThin
            isThin = bool(m.group(4))
        if valid and m.group(5):
            # set encoding.
            encoding = m.group(6)
            if encoding and encoding.endswith(','):
                # Leo 4.2 or after.
                encoding = encoding[:-1]
            if not g.isValidEncoding(encoding):  # pragma: no cover
                g.es_print("bad encoding in derived file:", encoding)
                valid = False
        if valid:
            end = m.group(8)  # closing delim
        if valid:
            at.encoding = encoding
            at.readVersion = readVersion
        return valid, new_df, start, end, isThin

    # @+node:ekr.20130911110233.11284: *5* at.readFileToUnicode & helpers
    def readFileToUnicode(self, fileName: str) -> Optional[str]:  # pragma: no cover
        """
        Carefully sets at.encoding, then uses at.encoding to convert the file
        to a unicode string.

        Sets at.encoding as follows:
        1. Use the BOM, if present. This unambiguously determines the encoding.
        2. Use the -encoding= field in the @+leo header, if present and valid.
        3. Otherwise, uses existing value of at.encoding.

        Returns the string, or None on failure.
        """
        at = self
        s: str
        s_bytes = at.openFileHelper(fileName)  # Catches all exceptions.
        # #1798.
        if not s_bytes:
            return None  # Not ''.
        e, s_bytes = g.stripBOM(s_bytes)
        if e:
            # The BOM determines the encoding unambiguously.
            s = g.toUnicode(s_bytes, encoding=e)
        else:
            # Get the encoding from the header, or the default encoding.
            s_temp = g.toUnicode(s_bytes, 'ascii', reportErrors=False)
            e = at.getEncodingFromHeader(fileName, s_temp)
            s = g.toUnicode(s_bytes, encoding=e)
        s = s.replace('\r\n', '\n')
        at.encoding = e
        at.initReadLine(s)
        return s

    # @+node:ekr.20130911110233.11285: *6* at.openFileHelper
    def openFileHelper(self, fileName: str) -> bytes:  # *not* str!
        """Open a file, reporting all exceptions."""
        at = self
        # #1798: return None as a flag on any error.
        s = None
        try:
            with open(fileName, 'rb') as f:
                s = f.read()
        except IOError:  # pragma: no cover
            if not g.unitTesting:
                at.error(f"can not open {fileName}")
        except Exception:  # pragma: no cover
            at.error(f"Exception reading {fileName}")
            g.es_exception()
        return s

    # @+node:ekr.20130911110233.11287: *6* at.getEncodingFromHeader
    def getEncodingFromHeader(self, fileName: str, s: str) -> str:
        """
        Return the encoding given in the @+leo sentinel, if the sentinel is
        present, or the previous value of at.encoding otherwise.
        """
        at = self
        if at.errors:  # pragma: no cover
            g.trace('can not happen: at.errors > 0', g.callers())
            e = at.encoding
            if g.unitTesting:
                assert False, g.callers()  # noqa
        else:
            at.initReadLine(s)
            old_encoding = at.encoding
            assert old_encoding
            at.encoding = None
            # Execute scanHeader merely to set at.encoding.
            at.scanHeader(fileName, giveErrors=False)
            e = at.encoding or old_encoding
        assert e
        return e

    # @+node:ekr.20041005105605.128: *5* at.readLine
    def readLine(self) -> str:
        """
        Read one line from file using the present encoding.
        Returns at.read_lines[at.read_i++]
        """
        # This is an old interface, now used only by at.scanHeader.
        # For now, it's not worth replacing.
        at = self
        if at.read_i < len(at.read_lines):
            s = at.read_lines[at.read_i]
            at.read_i += 1
            return s
        # Not an error.
        return ''  # pragma: no cover

    # @+node:ekr.20041005105605.129: *5* at.scanHeader
    def scanHeader(self, fileName: str, giveErrors: bool = True) -> tuple[list[str], bool, bool]:
        """
        Scan the @+leo sentinel, using the old readLine interface.

        Sets self.encoding, and self.start/endSentinelComment.

        Returns (firstLines,new_df,isThinDerivedFile) where:
        firstLines        contains all @first lines,
        new_df            is True if we are reading a new-format derived file.
        isThinDerivedFile is True if the file is an @thin file.
        """
        at = self
        new_df, isThinDerivedFile = False, False
        firstLines: list[str] = []  # The lines before @+leo.
        s = self.scanFirstLines(firstLines)
        valid = len(s) > 0
        if valid:
            valid, new_df, start, end, isThinDerivedFile = at.parseLeoSentinel(s)
        if valid:
            at.startSentinelComment = start
            at.endSentinelComment = end
        elif giveErrors:  # pragma: no cover
            at.error(f"No @+leo sentinel in: {fileName}")
            g.trace(g.callers())
        return firstLines, new_df, isThinDerivedFile

    # @+node:ekr.20041005105605.130: *6* at.scanFirstLines
    def scanFirstLines(self, firstLines: list[str]) -> str:  # pragma: no cover
        """
        Append all lines before the @+leo line to firstLines.

        Empty lines are ignored because empty @first directives are
        ignored.

        We can not call sentinelKind here because that depends on the comment
        delimiters we set here.
        """
        at = self
        s = at.readLine()
        while s and s.find("@+leo") == -1:
            firstLines.append(s)
            s = at.readLine()
        return s

    # @+node:ekr.20050103163224: *5* at.scanHeaderForThin (import code)
    def scanHeaderForThin(self, fileName: str) -> bool:  # pragma: no cover
        """
        Return true if the derived file is a thin file.

        This is a kludgy method used only by the import code."""
        at = self
        # Set at.encoding, regularize whitespace and call at.initReadLines.
        at.readFileToUnicode(fileName)
        # scanHeader uses at.readline instead of its args.
        # scanHeader also sets at.encoding.
        junk1, junk2, isThin = at.scanHeader(None)
        return isThin

    # @+node:ekr.20041005105605.132: *3* at.Writing
    # @+node:ekr.20041005105605.133: *4* Writing (top level)
    # @+node:ekr.20190111153551.1: *5* at.commands
    # @+node:ekr.20070806105859: *6* at.writeAtAutoNodes
    @cmd('write-at-auto-nodes')
    def writeAtAutoNodes(self, event: LeoKeyEvent = None) -> None:  # pragma: no cover
        """Write all @auto nodes in the selected outline."""
        at, c, p = self, self.c, self.c.p
        c.init_error_dialogs()
        after, found = p.nodeAfterTree(), False
        while p and p != after:
            if p.isAtAutoNode() and not p.isAtIgnoreNode():
                ok = at.writeOneAtAutoNode(p)
                if ok:
                    found = True
                    p.moveToNodeAfterTree()
                else:
                    p.moveToThreadNext()
            else:
                p.moveToThreadNext()
        if g.unitTesting:
            return
        if found:
            g.es("finished")
        else:
            g.es("no @auto nodes in the selected tree")
        c.raise_error_dialogs(kind='write')

    # @+node:ekr.20220120072251.1: *6* at.writeDirtyAtAutoNodes
    @cmd('write-dirty-at-auto-nodes')  # pragma: no cover
    def writeDirtyAtAutoNodes(self, event: LeoKeyEvent = None) -> None:
        """Write all dirty @auto nodes in the selected outline."""
        at, c, p = self, self.c, self.c.p
        c.init_error_dialogs()
        after, found = p.nodeAfterTree(), False
        while p and p != after:
            if p.isAtAutoNode() and not p.isAtIgnoreNode() and p.isDirty():
                ok = at.writeOneAtAutoNode(p)
                if ok:
                    found = True
                    p.moveToNodeAfterTree()
                else:
                    p.moveToThreadNext()
            else:
                p.moveToThreadNext()
        if g.unitTesting:
            return
        if found:
            g.es("finished")
        else:
            g.es("no dirty @auto nodes in the selected tree")
        c.raise_error_dialogs(kind='write')

    # @+node:ekr.20080711093251.3: *6* at.writeAtShadowNodes
    @cmd('write-at-shadow-nodes')
    def writeAtShadowNodes(self, event: LeoKeyEvent = None) -> bool:  # pragma: no cover
        """Write all @shadow nodes in the selected outline."""
        at, c, p = self, self.c, self.c.p
        c.init_error_dialogs()
        after, found = p.nodeAfterTree(), False
        while p and p != after:
            if p.atShadowFileNodeName() and not p.isAtIgnoreNode():
                ok = at.writeOneAtShadowNode(p)
                if ok:
                    found = True
                    g.blue(f"wrote {p.atShadowFileNodeName()}")
                    p.moveToNodeAfterTree()
                else:
                    p.moveToThreadNext()
            else:
                p.moveToThreadNext()
        if g.unitTesting:
            return found
        if found:
            g.es("finished")
        else:
            g.es("no @shadow nodes in the selected tree")
        c.raise_error_dialogs(kind='write')
        return found

    # @+node:ekr.20220120072917.1: *6* at.writeDirtyAtShadowNodes
    @cmd('write-dirty-at-shadow-nodes')
    def writeDirtyAtShadowNodes(self, event: LeoKeyEvent = None) -> bool:  # pragma: no cover
        """Write all @shadow nodes in the selected outline."""
        at, c, p = self, self.c, self.c.p
        c.init_error_dialogs()
        after, found = p.nodeAfterTree(), False
        while p and p != after:
            if p.atShadowFileNodeName() and not p.isAtIgnoreNode() and p.isDirty():
                ok = at.writeOneAtShadowNode(p)
                if ok:
                    found = True
                    g.blue(f"wrote {p.atShadowFileNodeName()}")
                    p.moveToNodeAfterTree()
                else:
                    p.moveToThreadNext()
            else:
                p.moveToThreadNext()
        if g.unitTesting:
            return found
        if found:
            g.es("finished")
        else:
            g.es("no dirty @shadow nodes in the selected tree")
        c.raise_error_dialogs(kind='write')
        return found

    # @+node:ekr.20041005105605.157: *5* at.putFile
    def putFile(self, root: Position, fromString: str = '', sentinels: bool = True) -> None:
        """Write the contents of the file to the output stream."""
        at = self
        s = fromString if fromString else root.v.b
        root.clearAllVisitedInTree()
        at.putAtFirstLines(s)
        at.putOpenLeoSentinel("@+leo-ver=5")
        at.putInitialComment()
        at.putOpenNodeSentinel(root)
        at.putBody(root, fromString=fromString)
        # The -leo sentinel is required to handle @last.
        at.putSentinel("@-leo")
        root.setVisited()
        at.putAtLastLines(s)

    # @+node:ekr.20041005105605.147: *5* at.writeAll & helpers
    def writeAll(self, all: bool = False, dirty: bool = False) -> None:
        """Write @file nodes in all or part of the outline"""
        at = self
        # This is the *only* place where these are set.
        # promptForDangerousWrite sets cancelFlag only if canCancelFlag is True.
        at.unchangedFiles = 0
        at.canCancelFlag = True
        at.cancelFlag = False
        at.yesToAll = False
        files, root = at.findFilesToWrite(all)
        for p in files:
            try:
                at.writeAllHelper(p, root)
            except Exception:  # pragma: no cover
                at.internalWriteError(p)
        # Make *sure* these flags are cleared for other commands.
        at.canCancelFlag = False
        at.cancelFlag = False
        at.yesToAll = False
        # Say the command is finished.
        at.reportEndOfWrite(files, all, dirty)
        # #2338: Never call at.saveOutlineIfPossible().

    # @+node:ekr.20190108052043.1: *6* at.findFilesToWrite
    def findFilesToWrite(self, force: bool) -> tuple[list[Position], Position]:  # pragma: no cover
        """
        Return a list of files to write.
        We must do this in a prepass, so as to avoid errors later.
        """
        trace = 'save' in g.app.debug and not g.unitTesting
        if trace:
            g.trace(f"writing *{'selected' if force else 'all'}* files")
        c = self.c
        if force:
            # The Write @<file> Nodes command.
            # Write all nodes in the selected tree.
            root = c.p
            p = c.p
            after = p.nodeAfterTree()
        else:
            # Write dirty nodes in the entire outline.
            root = c.rootPosition()
            p = c.rootPosition()
            after = None
        seen = set()
        files: list[Position] = []
        while p and p != after:
            if p.isAtLeoNode():
                p.moveToNodeAfterTree()
            elif p.isAtIgnoreNode() and not p.isAtAsisFileNode():
                # Honor @ignore in *body* text, but *not* in @asis nodes.
                if p.isAnyAtFileNode():
                    c.ignored_at_file_nodes.append(p.h)
                p.moveToNodeAfterTree()
            elif p.isAnyAtFileNode():
                data = p.v, c.fullPath(p)
                if data in seen:
                    if trace and force:
                        g.trace('Already seen', p.h)
                else:
                    seen.add(data)
                    files.append(p.copy())
                # Don't scan nested trees???
                p.moveToNodeAfterTree()
            else:
                p.moveToThreadNext()
        # When scanning *all* nodes, we only actually write dirty nodes.
        if not force:
            files = [z for z in files if z.isDirty()]
        if trace:
            g.printObj([z.h for z in files], tag='Files to be saved')
        return files, root

    # @+node:ekr.20190108053115.1: *6* at.internalWriteError
    def internalWriteError(self, p: Position) -> None:  # pragma: no cover
        """
        Fix bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
        Give a more urgent, more specific, more helpful message.
        """
        g.es_exception()
        g.es(f"Internal error writing: {p.h}", color='red')
        g.es('Please report this error to:', color='blue')
        g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
        g.es('Warning: changes to this file will be lost', color='red')
        g.es('unless you can save the file successfully.', color='red')

    # @+node:ekr.20190108112519.1: *6* at.reportEndOfWrite
    def reportEndOfWrite(
        self, files: list[Position], all: bool, dirty: bool
    ) -> None:  # pragma: no cover
        at = self
        if g.unitTesting:
            return
        if files:
            n = at.unchangedFiles
            g.es(f"finished: {n} unchanged file{g.plural(n)}")
        elif all:
            g.warning("no @<file> nodes in the selected tree")
        elif dirty:
            g.es("no dirty @<file> nodes in the selected tree")

    # @+node:ekr.20041005105605.149: *6* at.writeAllHelper & helper
    def writeAllHelper(self, p: Position, root: Position) -> None:
        """
        Write one file for at.writeAll.

        Do *not* write @auto files unless p == root.

        This prevents the write-all command from needlessly updating
        the @persistence data, thereby annoyingly changing the .leo file.
        """
        at = self
        at.root = root
        if p.isAtLeoNode() or p.isAtIgnoreNode():  # pragma: no cover
            # Should have been handled in findFilesToWrite.
            g.trace(f"Can not happen: unexpected node: {p.h}")
            return
        try:
            at.writePathChanged(p)
        except IOError:  # pragma: no cover
            return
        table = (
            (p.isAtAsisFileNode, at.asisWrite),
            (p.isAtAutoNode, at.writeOneAtAutoNode),
            (p.isAtCleanNode, at.writeOneAtCleanNode),
            (p.isAtEditNode, at.writeOneAtEditNode),
            (p.isAtFileNode, at.writeOneAtFileNode),
            (p.isAtJupytextNode, at.writeOneAtJupytextNode),
            (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
            (p.isAtShadowFileNode, at.writeOneAtShadowNode),
            (p.isAtThinFileNode, at.writeOneAtFileNode),
        )
        for pred, func in table:
            if pred():
                func(p)
                break
        else:  # pragma: no cover
            g.trace(f"Can not happen: {p.h}")
            return
        #
        # Clear the dirty bits in all descendant nodes.
        # The persistence data may still have to be written.
        for p2 in p.self_and_subtree(copy=False):
            p2.v.clearDirty()

    # @+node:ekr.20190108105509.1: *7* at.writePathChanged
    def writePathChanged(self, p: Position) -> None:  # pragma: no cover
        """
        raise IOError if p's path has changed *and* user forbids the write.
        """
        at, c = self, self.c
        #
        # Suppress this message during save-as and save-to commands.
        if c.ignoreChangedPaths:
            return  # pragma: no cover
        oldPath = g.os_path_normcase(at.getPathUa(p))
        newPath = g.os_path_normcase(c.fullPath(p))
        try:  # #1367: samefile can throw an exception.
            changed = oldPath and not os.path.samefile(oldPath, newPath)
        except Exception:
            changed = True
        if not changed:
            return
        ok = at.promptForDangerousWrite(
            fileName=None,
            message=(
                f"{g.tr('path changed for %s' % (p.h))}\n"
                f"{g.tr('write this file anyway?')}"
            ),
        )  # fmt: skip
        if not ok:
            raise IOError
        at.setPathUa(p, newPath)  # Remember that we have changed paths.

    # @+node:ekr.20190109172025.1: *5* at.writeAtAutoContents
    def writeAtAutoContents(self, fileName: str, root: Position) -> str:  # pragma: no cover
        """Common helper for atAutoToString and writeOneAtAutoNode."""
        at, c = self, self.c
        # Dispatch the proper writer.
        junk, ext = g.os_path_splitext(fileName)
        writer = at.dispatch(ext, root)
        if writer:
            at.outputList = []
            writer(root)
            return '' if at.errors else ''.join(at.outputList)
        if root.isAtAutoRstNode():
            # An escape hatch: fall back to the theRst writer
            # if there is no rst writer plugin.
            at.outputFile = io.StringIO()
            ok = c.rstCommands.writeAtAutoFile(root, fileName, at.outputFile)
            if not ok:
                return ''
            s = at.outputFile.getvalue()
            at.outputFile.close()
            return s
        # leo 5.6: allow undefined section references in all @auto files.
        try:
            g.app.allow_undefined_refs = True
            at.outputList = []
            at.putFile(root, sentinels=False)
            return '' if at.errors else ''.join(at.outputList)
        except Exception:
            return None
        finally:
            g.app.allow_undefined_refs = False

    # @+node:ekr.20190111153522.1: *5* at.writeX...
    # @+node:ekr.20041005105605.154: *6* at.asisWrite & helper
    def asisWrite(self, root: Position) -> None:  # pragma: no cover
        at, c = self, self.c
        try:
            c.endEditing()
            c.init_error_dialogs()
            fileName = at.initWriteIvars(root)
            # #1450.
            if not fileName or not at.precheck(fileName, root):
                at.addToOrphanList(root)
                return

            try:
                g.doHook('before-writing-external-file', c=c, p=root)
            except Exception:
                # The hook must print an error message.
                return

            at.outputList = []
            for p in root.self_and_subtree(copy=False):
                at.writeAsisNode(p)
            if not at.errors:
                contents = ''.join(at.outputList)
                at.replaceFile(contents, at.encoding, fileName, root)
        except Exception:
            at.writeException(fileName, root)

    silentWrite = asisWrite  # Compatibility with old scripts.

    # @+node:ekr.20170331141933.1: *7* at.writeAsisNode
    def writeAsisNode(self, p: Position) -> None:  # pragma: no cover
        """Write the p's node to an @asis file."""
        at = self

        def put(s: str) -> None:
            """Append s to self.output_list."""
            # #1480: Avoid calling at.os().
            s = g.toUnicode(s, at.encoding, reportErrors=True)
            at.outputList.append(s)

        # Write the headline only if it starts with '@@'.

        s = p.h
        if g.match(s, 0, "@@"):
            s = s[2:]
            if s:
                put('\n')  # Experimental.
                put(s)
                put('\n')
        # Write the body.
        s = p.b
        if s:
            put(s)

    # @+node:ekr.20041005105605.151: *6* at.writeMissing & helper
    def writeMissing(self, p: Position) -> None:  # pragma: no cover
        at, c = self, self.c
        writtenFiles = False
        c.init_error_dialogs()
        # #1450.
        at.initWriteIvars(root=p.copy())
        p = p.copy()
        after = p.nodeAfterTree()
        while p and p != after:  # Don't use iterator.
            if p.isAtAsisFileNode() or (p.isAnyAtFileNode() and not p.isAtIgnoreNode()):
                fileName = p.anyAtFileNodeName()
                if fileName:
                    fileName = c.fullPath(p)  # #1914.
                    if at.precheck(fileName, p):
                        at.writeMissingNode(p)
                        writtenFiles = True
                    else:
                        at.addToOrphanList(p)
                p.moveToNodeAfterTree()
            elif p.isAtIgnoreNode():
                p.moveToNodeAfterTree()
            else:
                p.moveToThreadNext()
        if not g.unitTesting:
            if writtenFiles:
                g.es("finished")
            else:
                g.es("no @file node in the selected tree")
        c.raise_error_dialogs(kind='write')

    # @+node:ekr.20041005105605.152: *7* at.writeMissingNode
    def writeMissingNode(self, p: Position) -> None:  # pragma: no cover
        at = self
        table = (
            (p.isAtAsisFileNode, at.asisWrite),
            (p.isAtAutoNode, at.writeOneAtAutoNode),
            (p.isAtCleanNode, at.writeOneAtCleanNode),
            (p.isAtEditNode, at.writeOneAtEditNode),
            (p.isAtFileNode, at.writeOneAtFileNode),
            (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
            (p.isAtShadowFileNode, at.writeOneAtShadowNode),
            (p.isAtThinFileNode, at.writeOneAtFileNode),
        )
        for pred, func in table:
            if pred():
                func(p)
                return
        g.trace(f"Can not happen unknown @<file> kind: {p.h}")

    # @+node:ekr.20070806141607: *6* at.writeOneAtAutoNode & helpers
    def writeOneAtAutoNode(self, p: Position) -> bool:  # pragma: no cover
        """
        Write p, an @auto node.
        File indices *must* have already been assigned.
        Return True if the node was written successfully.
        """
        at, c = self, self.c
        root = p.copy()
        try:
            c.endEditing()
            if not p.atAutoNodeName():
                return False
            fileName = at.initWriteIvars(root)
            at.sentinels = False
            # #1450.
            if not fileName or not at.precheck(fileName, root):
                at.addToOrphanList(root)
                return False

            try:
                g.doHook('before-writing-external-file', c=c, p=p)
            except Exception:
                # The hook must print an error message.
                return False

            if c.persistenceController:
                c.persistenceController.update_before_write_foreign_file(root)
            contents = at.writeAtAutoContents(fileName, root)
            if contents is None:
                g.es("not written:", fileName)
                at.addToOrphanList(root)
                return False
            at.replaceFile(
                contents,
                at.encoding,
                fileName,
                root,
                ignoreBlankLines=root.isAtAutoRstNode(),
            )
            return True
        except Exception:
            at.writeException(fileName, root)
            return False

    # @+node:ekr.20140728040812.17993: *7* at.dispatch & helpers
    def dispatch(self, ext: str, p: Position) -> Optional[Callable]:  # pragma: no cover
        """Return the correct writer function for p, an @auto node."""
        at = self
        # Match @auto type before matching extension.
        return at.writer_for_at_auto(p) or at.writer_for_ext(ext)

    # @+node:ekr.20140728040812.17995: *8* at.writer_for_at_auto
    def writer_for_at_auto(self, root: Position) -> Optional[Callable]:  # pragma: no cover
        """A factory returning a writer function for the given kind of @auto directive."""
        at = self
        d = g.app.atAutoWritersDict
        for key in d:
            aClass = d.get(key)
            if aClass and g.match_word(root.h, 0, key):

                def writer_for_at_auto_cb(root: Position) -> Optional[str]:
                    try:
                        writer = aClass(at.c)  # noqa
                        s = writer.write(root)
                        return s
                    except Exception:
                        g.es_exception()
                        return None

                return writer_for_at_auto_cb
        return None

    # @+node:ekr.20140728040812.17997: *8* at.writer_for_ext
    def writer_for_ext(self, ext: str) -> Optional[Callable]:  # pragma: no cover
        """A factory returning a writer function for the given file extension."""
        at = self
        d = g.app.writersDispatchDict
        aClass = d.get(ext)
        if aClass:

            def writer_for_ext_cb(root: Position) -> Optional[str]:
                try:
                    return aClass(at.c).write(root)
                except Exception:
                    g.es_exception()
                    return None

            return writer_for_ext_cb

        return None

    # @+node:ekr.20210501064359.1: *6* at.writeOneAtCleanNode
    def writeOneAtCleanNode(self, root: Position) -> None:  # pragma: no cover
        """Write one @clean file..
        root is the position of an @clean node.
        """
        at, c = self, self.c
        try:
            c.endEditing()
            fileName = at.initWriteIvars(root)
            if not fileName or not at.precheck(fileName, root):
                return

            try:
                g.doHook('before-writing-external-file', c=c, p=root)
            except Exception:
                # The hook must print an error message.
                return

            at.outputList = []
            try:
                at.sentinels = False
                at.putFile(root, sentinels=False)
            finally:
                at.sentinels = True

            at.warnAboutOrphandAndIgnoredNodes()
            if at.errors:
                g.es("not written:", g.shortFileName(fileName))
                at.addToOrphanList(root)
            else:
                contents = ''.join(at.outputList)
                at.replaceFile(contents, at.encoding, fileName, root)
                # #4385: This and readOneAtCleanNode are the *only* two places that sets the `_mod_time` uA.
                root.v.u['_mod_time'] = g.os_path_getmtime(fileName)

        except Exception:
            at.writeException(fileName, root)

    # @+node:ekr.20090225080846.5: *6* at.writeOneAtEditNode
    def writeOneAtEditNode(self, p: Position) -> bool:  # pragma: no cover
        """Write one @edit node."""
        at, c = self, self.c
        root = p.copy()
        try:
            c.endEditing()
            c.init_error_dialogs()
            if not p.atEditNodeName():
                return False
            if p.hasChildren():
                g.error('@edit nodes must not have children')
                g.es('To save your work, convert @edit to @auto, @file or @clean')
                return False
            fileName = at.initWriteIvars(root)
            at.sentinels = False
            # #1450.
            if not fileName or not at.precheck(fileName, root):
                at.addToOrphanList(root)
                return False

            try:
                g.doHook('before-writing-external-file', c=c, p=root)
            except Exception:
                # The hook must print an error message.
                return False

            contents = ''.join(
                [s for s in g.splitLines(p.b) if at.directiveKind4(s, 0) == at.noDirective]
            )
            at.replaceFile(contents, at.encoding, fileName, root)
            c.raise_error_dialogs(kind='write')
            return True
        except Exception:
            at.writeException(fileName, root)
            return False

    # @+node:ekr.20210501075610.1: *6* at.writeOneAtFileNode
    def writeOneAtFileNode(self, root: Position) -> None:  # pragma: no cover
        """Write @file or @thin file."""
        at, c = self, self.c
        try:
            c.endEditing()
            fileName = at.initWriteIvars(root)
            at.sentinels = True
            if not fileName or not at.precheck(fileName, root):
                # Raise dialog warning of data loss.
                at.addToOrphanList(root)
                return

            try:
                g.doHook('before-writing-external-file', c=c, p=root)
            except Exception:
                # The hook must print an error message.
                return

            at.outputList = []
            at.putFile(root, sentinels=True)
            at.warnAboutOrphandAndIgnoredNodes()
            if at.errors:
                g.es("not written:", g.shortFileName(fileName))
                at.addToOrphanList(root)
            else:
                contents = ''.join(at.outputList)
                at.replaceFile(contents, at.encoding, fileName, root)
        except Exception:
            at.writeException(fileName, root)

    # @+node:ekr.20241023134114.1: *6* at.writeOneAtJupytextNode
    def writeOneAtJupytextNode(self, root: Position) -> None:  # pragma: no cover
        """
        p must be an @jupytext node.
        Write the corresponding .ipynb file from p and all p's descendants.

        This code is adapted from at.writeOneAtCleanNode.
        """
        at, c, p = self, self.c, self.c.p
        try:
            c.endEditing()
            fileName = at.initWriteIvars(root)
            at.sentinels = False

            # Prompt for dangerous write if the file exists.
            if not fileName or not at.precheck(fileName, root):
                return

            try:
                g.doHook('before-writing-external-file', c=c, p=p)
            except Exception:
                # The hook must print an error message.
                return

            # Write a minimal Jupyter file if the @jupytext tree is empty.
            if not root.b.strip() and not root.hasChildren():
                prefix_list = c.config.getData(
                    'jupyter-prefix', strip_comments=False, strip_data=False
                )
                if prefix_list:
                    prefix = ''.join(prefix_list)
                    root.b = prefix.strip() + '\n\n'
            at.outputList = []
            at.putFile(root, sentinels=False)
            at.warnAboutOrphandAndIgnoredNodes()
            if at.errors:
                g.es_print(f"not written: {fileName}")
                at.addToOrphanList(root)
            else:
                new_contents = ''.join(at.outputList)
                g.app.jupytextManager.write(c, root, new_contents)
                # Similar to at.replaceFile.
                c.setFileTimeStamp(fileName)
                at.rememberReadPath(fileName, root)
                g.es_print(f"wrote: {fileName}")
        except Exception:
            at.writeException(fileName, root)

    # @+node:ekr.20210501065352.1: *6* at.writeOneAtNosentNode
    def writeOneAtNosentNode(self, root: Position) -> None:  # pragma: no cover
        """Write one @nosent node.
        root is the position of an @<file> node.
        sentinels will be False for @clean and @nosent nodes.
        """
        at, c, p = self, self.c, self.c.p
        try:
            c.endEditing()
            fileName = at.initWriteIvars(root)
            at.sentinels = False
            if not fileName or not at.precheck(fileName, root):
                return

            try:
                g.doHook('before-writing-external-file', c=c, p=p)
            except Exception:
                # The hook must print an error message.
                return

            at.outputList = []
            at.putFile(root, sentinels=False)
            at.warnAboutOrphandAndIgnoredNodes()
            if at.errors:
                g.es("not written:", g.shortFileName(fileName))
                at.addToOrphanList(root)
            else:
                contents = ''.join(at.outputList)
                at.replaceFile(contents, at.encoding, fileName, root)
        except Exception:
            at.writeException(fileName, root)

    # @+node:ekr.20080711093251.5: *6* at.writeOneAtShadowNode & helper
    def writeOneAtShadowNode(self, p: Position, testing: bool = False) -> bool:  # pragma: no cover
        """
        Write p, an @shadow node.
        File indices *must* have already been assigned.

        testing: set by unit tests to suppress the call to at.precheck.
                 Testing is not the same as g.unitTesting.
        """
        at, c = self, self.c
        root = p.copy()
        x = c.shadowController
        try:
            c.endEditing()  # Capture the current headline.
            fn = p.atShadowFileNodeName()
            assert fn, p.h
            # A hack to support unknown extensions. May set c.target_language.
            self.adjustTargetLanguage(fn)
            full_path = c.fullPath(p)
            at.initWriteIvars(root)
            # Force python sentinels to suppress an error message.
            # The actual sentinels will be set below.
            at.endSentinelComment = None
            at.startSentinelComment = "#"
            # Make sure we can compute the shadow directory.
            private_fn = x.shadowPathName(full_path)
            if not private_fn:
                return False
            if not testing and not at.precheck(full_path, root):
                return False

            try:
                g.doHook('before-writing-external-file', c=c, p=p)
            except Exception:
                # The hook must print an error message.
                return False

            # Bug fix: Leo 4.5.1:
            # use x.markerFromFileName to force the delim to match
            # what is used in x.propagate_changes.
            marker = x.markerFromFileName(full_path)
            at.startSentinelComment, at.endSentinelComment = marker.getDelims()
            if g.unitTesting:
                ivars_dict = g.getIvarsDict(at)
            #
            # Write the public and private files to strings.

            def put(sentinels: bool) -> str:
                at.outputList = []
                at.sentinels = sentinels
                at.putFile(root, sentinels=sentinels)
                return '' if at.errors else ''.join(at.outputList)

            at.public_s = put(False)
            at.private_s = put(True)
            at.warnAboutOrphandAndIgnoredNodes()
            if g.unitTesting:
                exceptions = ('public_s', 'private_s', 'sentinels', 'outputList')
                assert g.checkUnchangedIvars(at, ivars_dict, exceptions), 'writeOneAtShadowNode'
            if not at.errors:
                # Write the public and private files.
                # makeShadowDirectory takes a *public* file name.
                x.makeShadowDirectory(full_path)
                x.replaceFileWithString(at.encoding, private_fn, at.private_s)
                x.replaceFileWithString(at.encoding, full_path, at.public_s)
            at.checkPythonCode(contents=at.private_s, fileName=full_path, root=root)
            if at.errors:
                g.error("not written:", full_path)
                at.addToOrphanList(root)
            else:
                root.clearDirty()
            return not at.errors
        except Exception:
            at.writeException(full_path, root)
            return False

    # @+node:ekr.20080819075811.13: *7* at.adjustTargetLanguage
    def adjustTargetLanguage(self, fn: str) -> None:  # pragma: no cover
        """Use the language implied by fn's extension if
        there is a conflict between it and c.target_language."""
        at = self
        c = at.c
        junk, ext = g.os_path_splitext(fn)
        if ext:
            if ext.startswith('.'):
                ext = ext[1:]
            language = g.app.extension_dict.get(ext)
            if language:
                c.target_language = language
            else:
                # An unknown language.
                # Use the default language, **not** 'unknown_language'
                pass

    # @+node:ekr.20190111153506.1: *5* at.XToString
    # @+node:ekr.20190109160056.1: *6* at.atAsisToString
    def atAsisToString(self, root: Position) -> str:  # pragma: no cover
        """Write the @asis node to a string."""
        at, c = self, self.c
        c.endEditing()
        fileName = at.initWriteIvars(root)
        try:
            at.outputList = []
            for p in root.self_and_subtree(copy=False):
                at.writeAsisNode(p)
            return '' if at.errors else ''.join(at.outputList)
        except Exception:
            at.writeException(fileName, root)
            return ''

    # @+node:ekr.20190109160056.2: *6* at.atAutoToString
    def atAutoToString(self, root: Position) -> str:  # pragma: no cover
        """Write the root @auto node to a string, and return it."""
        at, c = self, self.c
        try:
            c.endEditing()
            fileName = at.initWriteIvars(root)
            at.sentinels = False
            # #1450.
            if not fileName:
                at.addToOrphanList(root)
                return ''
            return at.writeAtAutoContents(fileName, root) or ''
        except Exception:
            at.writeException(fileName, root)
            return ''

    # @+node:ekr.20230804025627.1: *6* at.atCleanToString
    def atCleanToString(self, root: Position) -> str:  # pragma: no cover
        """Write one @clean node to a string."""
        at, c = self, self.c
        c.endEditing()
        fileName = at.initWriteIvars(root)
        try:
            at.sentinels = False
            at.outputList = []
            at.putFile(root, sentinels=False)
            return '' if at.errors else ''.join(at.outputList)
        except Exception:
            at.writeException(fileName, root)
            return ''

    # @+node:ekr.20190109160056.3: *6* at.atEditToString
    def atEditToString(self, root: Position) -> str:  # pragma: no cover
        """Write one @edit node."""
        at, c = self, self.c
        try:
            c.endEditing()
            if root.hasChildren():
                g.error('@edit nodes must not have children')
                g.es('To save your work, convert @edit to @auto, @file or @clean')
                return ''
            fileName = at.initWriteIvars(root)
            at.sentinels = False
            # #1450.
            if not fileName:
                at.addToOrphanList(root)
                return ''
            contents = ''.join(
                [s for s in g.splitLines(root.b) if at.directiveKind4(s, 0) == at.noDirective]
            )
            return contents
        except Exception:
            at.writeException(fileName, root)
            return ''

    # @+node:ekr.20190109142026.1: *6* at.atFileToString
    def atFileToString(self, root: Position, sentinels: bool = True) -> str:  # pragma: no cover
        """Write an external file to a string, and return its contents."""
        at, c = self, self.c
        try:
            c.endEditing()
            at.initWriteIvars(root)
            at.sentinels = sentinels
            at.outputList = []
            at.putFile(root, sentinels=sentinels)
            assert root == at.root, 'write'
            contents = '' if at.errors else ''.join(at.outputList)
            return contents
        except Exception:
            at.exception("exception preprocessing script")
            root.v._p_changed = True
            return ''

    # @+node:ekr.20050506084734: *6* at.stringToString
    def stringToString(
        self,
        root: Position,
        s: str,
        forcePythonSentinels: bool = True,
        sentinels: bool = True,
    ) -> str:  # pragma: no cover
        """
        Write an external file from a string.

        This is at.write specialized for scripting.
        """
        at, c = self, self.c
        try:
            c.endEditing()
            at.initWriteIvars(root)
            if forcePythonSentinels:
                at.endSentinelComment = None
                at.startSentinelComment = "#"
                at.language = "python"
            at.sentinels = sentinels
            at.outputList = []
            at.putFile(root, fromString=s, sentinels=sentinels)
            contents = '' if at.errors else ''.join(at.outputList)
            # Major bug: failure to clear this wipes out headlines!
            #            Sometimes this causes slight problems...
            if root:
                root.v._p_changed = True
            return contents
        except Exception:
            at.exception("exception preprocessing script")
            return ''

    # @+node:ekr.20041005105605.160: *4* Writing helpers
    # @+node:ekr.20041005105605.161: *5* at.putBody & helper
    def putBody(self, p: Position, fromString: str = '') -> bool:
        """
        Generate the body enclosed in sentinel lines.
        Return True if the body contains an @others line.
        """
        at = self
        s = fromString if fromString else p.b

        # Make sure v is never expanded again.
        # Suppress orphans check.
        p.v.setVisited()

        # #1048 & #1037: regularize most trailing whitespace.
        if s and (at.sentinels or at.force_newlines_in_at_nosent_bodies):
            if not s.endswith('\n'):
                s = s + '\n'

        i = 0
        status = LeoIOStatus()
        while i < len(s):
            next_i = g.skip_line(s, i)
            assert next_i > i, 'putBody'
            kind = at.directiveKind4(s, i)
            at.putLine(i, kind, p, s, status)
            i = next_i
        if not status.in_code:
            at.putEndDocLine()
        return status.has_at_others

    # @+node:ekr.20041005105605.163: *6* at.putLine
    def putLine(self, i: int, kind: int, p: Position, s: str, status: LeoIOStatus) -> None:
        """Put the line at s[i:] of the given kind, updating the status."""
        at = self
        if kind == at.noDirective:
            if status.in_code:
                # Important: the so-called "name" must include brackets.
                name, n1, n2 = at.findSectionName(s, i, p)
                if name:
                    at.putRefLine(s, i, n1, n2, name, p)
                else:
                    at.putCodeLine(s, i)
            else:
                at.putDocLine(s, i)
        elif kind in (at.docDirective, at.atDirective):
            if not status.in_code:
                # Bug fix 12/31/04: handle adjacent doc parts.
                at.putEndDocLine()
            at.putStartDocLine(s, i, kind)
            status.in_code = False
        elif kind in (at.cDirective, at.codeDirective):
            # Only @c and @code end a doc part.
            if not status.in_code:
                at.putEndDocLine()
            at.putDirective(s, i, p)
            status.in_code = True
        elif kind == at.allDirective:
            if status.in_code:
                if p == self.root:
                    at.putAtAllLine(s, i, p)
                else:
                    at.error(f"@all not valid in: {p.h}")  # pragma: no cover
            else:
                at.putDocLine(s, i)
        elif kind == at.othersDirective:
            if status.in_code:
                if status.has_at_others:
                    at.error(f"multiple @others in: {p.h}")  # pragma: no cover
                else:
                    at.putAtOthersLine(s, i, p)
                    status.has_at_others = True
            else:
                at.putDocLine(s, i)
        elif kind == at.startVerbatim:  # pragma: no cover
            # Fix bug 778204: @verbatim not a valid Leo directive.
            if g.unitTesting:
                # A hack: unit tests for @shadow use @verbatim as a kind of directive.
                pass
            else:
                at.error(f"@verbatim is not a Leo directive: {p.h}")
        elif kind == at.miscDirective:
            # Fix bug 583878: Leo should warn about @comment/@delims clashes.
            if g.match_word(s, i, '@comment'):
                status.at_comment_seen = True
            elif g.match_word(s, i, '@delims'):
                status.at_delims_seen = True
            if (
                status.at_comment_seen and status.at_delims_seen and not status.at_warning_given
            ):  # pragma: no cover
                status.at_warning_given = True
                at.error(f"@comment and @delims in node {p.h}")
            at.putDirective(s, i, p)
        else:
            at.error(f"putBody: can not happen: unknown directive kind: {kind}")  # pragma: no cover

    # @+node:ekr.20041005105605.164: *5* writing code lines...
    # @+node:ekr.20041005105605.165: *6* at: @all
    # @+node:ekr.20041005105605.166: *7* at.putAtAllLine
    def putAtAllLine(self, s: str, i: int, p: Position) -> None:
        """Put the expansion of @all."""
        at = self
        j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
        k = g.skip_to_end_of_line(s, i)
        at.putLeadInSentinel(s, i, j)
        at.indent += delta
        # s[j:k] starts with '@all'
        at.putSentinel("@+" + s[j + 1 : k].strip())
        for child in p.children():
            at.putAtAllChild(child)
        at.putSentinel("@-all")
        at.indent -= delta

    # @+node:ekr.20041005105605.167: *7* at.putAtAllBody
    def putAtAllBody(self, p: Position) -> None:
        """Generate the body enclosed in sentinel lines."""
        at = self
        s = p.b
        # Make sure v is never expanded again.
        # Suppress orphans check.
        p.v.setVisited()
        if at.sentinels and s and s[-1] != '\n':
            s = s + '\n'
        i = 0
        # Leo 6.6. This code never changes at.in_code status!
        while i < len(s):
            next_i = g.skip_line(s, i)
            assert next_i > i
            at.putCodeLine(s, i)
            i = next_i

    # @+node:ekr.20041005105605.169: *7* at.putAtAllChild
    def putAtAllChild(self, p: Position) -> None:
        """
        This code puts only the first of two or more cloned siblings, preceding
        the clone with an @clone n sentinel.

        This is a debatable choice: the cloned tree appears only once in the
        external file. This should be benign; the text created by @all is
        likely to be used only for recreating the outline in Leo. The
        representation in the derived file doesn't matter much.
        """
        at = self
        # Suppress warnings about @file nodes.
        at.putOpenNodeSentinel(p, inAtAll=True)
        at.putAtAllBody(p)
        for child in p.children():
            at.putAtAllChild(child)  # pragma: no cover (recursive call)

    # @+node:ekr.20041005105605.170: *6* at: @others
    # @+node:ekr.20041005105605.173: *7* at.putAtOthersLine & helper
    def putAtOthersLine(self, s: str, i: int, p: Position) -> None:
        """Put the expansion of @others."""
        at = self
        j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
        k = g.skip_to_end_of_line(s, i)
        at.putLeadInSentinel(s, i, j)
        at.indent += delta
        # s[j:k] starts with '@others'
        # Never write lws in new sentinels.
        at.putSentinel("@+" + s[j + 1 : k].strip())
        for child in p.children():
            p = child.copy()
            after = p.nodeAfterTree()
            while p and p != after:
                if at.validInAtOthers(p):
                    at.putOpenNodeSentinel(p)
                    at_others_flag = at.putBody(p)
                    if at_others_flag:
                        p.moveToNodeAfterTree()
                    else:
                        p.moveToThreadNext()
                else:
                    p.moveToNodeAfterTree()
        # This is the same in both old and new sentinels.
        at.putSentinel("@-others")
        at.indent -= delta

    # @+node:ekr.20041005105605.171: *8* at.validInAtOthers
    def validInAtOthers(self, p: Position) -> bool:
        """
        Return True if p should be included in the expansion of the @others
        directive in the body text of p's parent.
        """
        at = self
        i = g.skip_ws(p.h, 0)
        isSection, junk = at.isSectionName(p.h, i)
        if isSection:
            return False  # A section definition node.
        if at.sentinels:
            # @verbatim
            # @ignore must not stop expansion here!
            return True
        if p.isAtIgnoreNode():  # pragma: no cover
            g.error('did not write @ignore node', p.v.h)
            return False
        return True

    # @+node:ekr.20041005105605.199: *6* at.findSectionName
    def findSectionName(self, s: str, i: int, p: Position) -> tuple[str, int, int]:
        """
        Return n1, n2 representing a section name.

        Return the reference, *including* brackets.
        """
        at = self

        def is_space(i1: int, i2: int) -> bool:
            """A replacement for s[i1 : i2] that doesn't create any substring."""
            return i == j or all(s[z] in ' \t\n' for z in range(i1, i2))

        end = s.find('\n', i)
        j = len(s) if end == -1 else end
        # Careful: don't look beyond the end of the line!
        if end == -1:
            n1 = s.find(at.section_delim1, i)
            n2 = s.find(at.section_delim2, i)
        else:
            n1 = s.find(at.section_delim1, i, end)
            n2 = s.find(at.section_delim2, i, end)
        n3 = n2 + len(at.section_delim2)
        if -1 < n1 < n2:  # A *possible* section reference.
            if is_space(i, n1) and is_space(n3, j):  # A *real* section reference.
                return s[n1:n3], n1, n3
            # An apparent section reference.
            if 'sections' in g.app.debug and not g.unitTesting:  # pragma: no cover
                i1, i2 = g.getLine(s, i)
                g.es_print('Ignoring apparent section reference:', color='red')
                g.es_print('Node: ', p.h)
                g.es_print('Line: ', s[i1:i2].rstrip())
        return None, 0, 0

    # @+node:ekr.20041005105605.174: *6* at.putCodeLine
    def putCodeLine(self, s: str, i: int) -> None:
        """Put a normal code line."""
        at = self
        j = g.skip_line(s, i)
        k = g.skip_ws(s, i)
        line = s[i:j]

        def put_verbatim_sentinel() -> None:
            """Put an @verbatim sentinel."""
            if at.root.isAtCleanNode():
                # #2996. Adding an @verbatim sentinel interferes with the @clean algorithm.
                return  # # pragma: no cover (defensive)
            ws = s[i:k]
            self.putIndent(len(ws))
            self.putSentinel("@verbatim")

        # Put an @verbatim sentinel if the next line looks like another sentinel.
        if at.language == 'python':  # New in Leo 6.7.2.
            # Python sentinels *only* may contain a space between '#' and '@'
            if g.match(s, k, '#@') or g.match(s, k, '# @'):
                put_verbatim_sentinel()
        elif g.match(s, k, self.startSentinelComment + "@"):
            put_verbatim_sentinel()

        # Don't put any whitespace in otherwise blank lines.
        if len(line) > 1:  # Preserve *anything* the user puts on the line!!!
            at.putIndent(at.indent, line)
            if line[-1:] == '\n':
                at.os(line[:-1])
                at.onl()
            else:
                at.os(line)
        elif line and line[-1] == '\n':
            at.onl()
        elif line:
            at.os(line)  # Bug fix: 2013/09/16
        else:
            g.trace('Can not happen: completely empty line')  # pragma: no cover

    # @+node:ekr.20041005105605.176: *6* at.putRefLine
    def putRefLine(self, s: str, i: int, n1: int, n2: int, name: str, p: Position) -> None:
        """
        Put a line containing one or more references.

        Important: the so-called name *must* include brackets.
        """
        at = self
        ref = g.findReference(name, p)
        if ref:
            junk, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
            at.putLeadInSentinel(s, i, n1)
            at.indent += delta
            at.putSentinel("@+" + name)
            at.putOpenNodeSentinel(ref)
            at.putBody(ref)
            at.putSentinel("@-" + name)
            at.indent -= delta
            return
        if g.app.allow_undefined_refs:  # pragma: no cover
            p.v.setVisited()  # #2311
            # Allow apparent section reference: just write the line.
            at.putCodeLine(s, i)
        else:  # pragma: no cover
            # Do give this error even if unit testing.
            at.writeError(
                f"undefined section: {g.truncate(name, 60)}\n"
                f"  referenced from: {g.truncate(p.h, 60)}"
            )  # fmt: skip

    # @+node:ekr.20041005105605.180: *5* writing doc lines...
    # @+node:ekr.20041005105605.181: *6* at.putBlankDocLine
    def putBlankDocLine(self) -> None:
        at = self
        if not at.endSentinelComment:
            at.putIndent(at.indent)
            at.os(at.startSentinelComment)
            # #1496: Retire the @doc convention.
            #        Remove the blank.
            # at.oblank()
        at.onl()

    # @+node:ekr.20041005105605.183: *6* at.putDocLine
    def putDocLine(self, s: str, i: int) -> None:
        """Handle one line of a doc part."""
        at = self
        j = g.skip_line(s, i)
        s = s[i:j]
        #
        # #1496: Retire the @doc convention:
        #        Strip all trailing ws here.
        if not s.strip():
            # A blank line.
            at.putBlankDocLine()
            return

        # Write the line as it is.
        at.putIndent(at.indent)
        if not at.endSentinelComment:
            at.os(at.startSentinelComment)
            # #1496: Retire the @doc convention.
            #        Leave this blank. The line is not blank.
            at.oblank()
        at.os(s)
        if not s.endswith('\n'):
            at.onl()  # pragma: no cover

    # @+node:ekr.20041005105605.185: *6* at.putEndDocLine
    def putEndDocLine(self) -> None:
        """Write the conclusion of a doc part."""
        at = self
        # Put the closing delimiter if we are using block comments.
        if at.endSentinelComment:
            at.putIndent(at.indent)
            at.os(at.endSentinelComment)
            at.onl()  # Note: no trailing whitespace.

    # @+node:ekr.20041005105605.182: *6* at.putStartDocLine
    def putStartDocLine(self, s: str, i: int, kind: int) -> None:
        """Write the start of a doc part."""
        at = self
        sentinel = "@+doc" if kind == at.docDirective else "@+at"
        directive = "@doc" if kind == at.docDirective else "@"
        # Put whatever follows the directive in the sentinel.
        # Skip past the directive.
        i += len(directive)
        j = g.skip_to_end_of_line(s, i)
        follow = s[i:j]
        # Put the opening @+doc or @-doc sentinel, including whatever follows the directive.
        at.putSentinel(sentinel + follow)
        # Put the opening comment if we are using block comments.
        if at.endSentinelComment:
            at.putIndent(at.indent)
            at.os(at.startSentinelComment)
            at.onl()

    # @+node:ekr.20041005105605.187: *4* Writing sentinels...
    # @+node:ekr.20041005105605.188: *5* at.nodeSentinelText & helper
    def nodeSentinelText(self, p: Position) -> str:
        """Return the text of a @+node or @-node sentinel for p."""
        at = self
        h = at.removeCommentDelims(p)
        if getattr(at, 'at_shadow_test_hack', False):  # pragma: no cover
            # A hack for @shadow unit testing.
            # see AtShadowTestCase.makePrivateLines.
            return h
        gnx = p.v.fileIndex
        level = 1 + p.level() - self.root.level()
        if level > 2:
            return f"{gnx}: *{level}* {h}"
        return f"{gnx}: {'*' * level} {h}"

    # @+node:ekr.20041005105605.189: *6* at.removeCommentDelims
    def removeCommentDelims(self, p: Position) -> str:
        """
        If the present @language/@comment settings do not specify a single-line comment
        we remove all block comment delims from h. This prevents headline text from
        interfering with the parsing of node sentinels.
        """
        at = self
        start = at.startSentinelComment
        end = at.endSentinelComment
        h = p.h
        if end:
            h = h.replace(start, "")
            h = h.replace(end, "")
        return h

    # @+node:ekr.20041005105605.190: *5* at.putLeadInSentinel
    def putLeadInSentinel(self, s: str, i: int, j: int) -> None:
        """
        i points at the start of a line.
        j points at @others or a section reference.
        """
        at = self
        if i == j:
            return  # The @others or ref starts a line.
        k = g.skip_ws(s, i)
        if j == k:
            pass
        else:
            self.putIndent(at.indent)  # 1/29/04: fix bug reported by Dan Winkler.
            at.os(s[i:j])
            at.onl_sent()

    # @+node:ekr.20041005105605.192: *5* at.putOpenLeoSentinel 4.x
    def putOpenLeoSentinel(self, s: str) -> None:
        """Write @+leo sentinel."""
        at = self
        if at.sentinels or g.app.force_at_auto_sentinels:
            s = s + "-thin"
            encoding = at.encoding.lower()
            if encoding != "utf-8":  # pragma: no cover
                # New in 4.2: encoding fields end in ",."
                s = s + f"-encoding={encoding},."
            at.putSentinel(s)

    # @+node:ekr.20041005105605.193: *5* at.putOpenNodeSentinel
    def putOpenNodeSentinel(self, p: Position, inAtAll: bool = False) -> None:
        """Write @+node sentinel for p."""
        # Note: lineNumbers.py overrides this method.
        at = self
        if not inAtAll and p.isAtFileNode() and p != at.root:  # pragma: no cover
            at.writeError("@file not valid in: " + p.h)
            return
        s = at.nodeSentinelText(p)
        at.putSentinel("@+node:" + s)
        # Leo 4.7: we never write tnodeLists.

    # @+node:ekr.20041005105605.194: *5* at.putSentinel (applies cweb hack) 4.x
    def putSentinel(self, s: str) -> None:
        """
        Write a sentinel whose text is s, applying the CWEB hack if needed.

        This method outputs all sentinels.
        """
        at = self
        if at.sentinels or g.app.force_at_auto_sentinels:
            at.putIndent(at.indent)
            at.os(at.startSentinelComment)
            # Blacken python sentinels.
            if self.language == 'python':
                at.os(' ')
            # Apply the cweb hack to s:
            #   If the opening comment delim ends in '@',
            #   double all '@' signs except the first.
            start = at.startSentinelComment
            if start and start[-1] == '@':
                s = s.replace('@', '@@')[1:]
            at.os(s)
            if at.endSentinelComment:
                at.os(at.endSentinelComment)
            at.onl()

    # @+node:ekr.20041005105605.196: *4* Writing utils...
    # @+node:ekr.20181024134823.1: *5* at.addToOrphanList
    def addToOrphanList(self, root: Position) -> None:  # pragma: no cover
        """Mark the root as erroneous for c.raise_error_dialogs()."""
        c = self.c
        # Fix #1050:
        root.setOrphan()
        c.orphan_at_file_nodes.append(root.h)

    # @+node:ekr.20090514111518.5661: *5* at.checkPythonCode & helpers
    def checkPythonCode(
        self, contents: str, fileName: str, root: Position
    ) -> None:  # pragma: no cover
        """Perform python-related checks on root."""
        if not g.app.log:
            return  # We are auto-saving.
        is_python = fileName and fileName.endswith(('py', 'pyw'))
        if g.unitTesting or not contents or not is_python:
            return
        ok = True
        if self.checkPythonCodeOnWrite:  # First
            ok = self.checkPythonSyntax(root, contents)
        if ok and self.beautifyOnWrite:  # Second.
            ok = self.runRuffFormat(contents, root, fileName)
        if ok and self.runPyFlakesOnWrite:  # Creates clickable links.
            ok = self.runPyflakes(root)
        if ok and self.runFlake8OnWrite:  # Does *not* create clickable links.
            ok = self.runFlake8(root)
        if ok and self.runPylintOnWrite:
            ok = self.runPylint(root)
        if not ok:
            g.app.syntax_error_files.append(g.shortFileName(fileName))

    # @+node:ekr.20090514111518.5663: *6* at.checkPythonSyntax
    def checkPythonSyntax(self, p: Position, body: str) -> bool:
        """Check the syntax of the given node."""
        try:
            body = body.replace('\r', '')
            fn = f"<node: {p.h}>"
            compile(body + '\n', fn, 'exec')
            return True
        except SyntaxError:  # pragma: no cover
            if not g.unitTesting:
                self.syntaxError(p, body)
        except Exception:  # pragma: no cover
            g.trace("unexpected exception")
            g.es_exception()
        return False

    # @+node:ekr.20090514111518.5666: *7* at.syntaxError (leoAtFile)
    def syntaxError(self, p: Position, body: str) -> None:  # pragma: no cover
        """Report a syntax error."""
        g.error(f"Syntax error in: {p.h}")
        typ, val, tb = sys.exc_info()
        message = hasattr(val, 'message') and val.message
        if message:
            g.es_print(message)
        if val is None:
            return
        lines = g.splitLines(body)
        n = val.lineno
        offset = val.offset or 0
        if n is None:
            return
        i = val.lineno - 1
        for j in range(max(0, i - 2), min(i + 2, len(lines) - 1)):
            line = lines[j].rstrip()
            if j == i:
                unl = p.get_UNL()
                g.es_print(f"{j + 1:5}:* {line}", nodeLink=f"{unl}::-{j + 1}")  # Global line.
                g.es_print(' ' * (7 + offset) + '^')
            else:
                g.es_print(f"{j + 1:5}: {line}")

    # @+node:ekr.20240926044644.1: *6* at.runRuffFormat
    def runRuffFormat(self, contents: str, root: Position, filename: str) -> bool:
        """
        Run ruff format on the selected position.
        Return True if all went well.
        """
        c, p, u = self.c, self.c.p, self.c.undoer
        efc = g.app.externalFilesController
        if not os.path.exists(filename):
            return False

        # Copy the old p.b. c.refresh_from_disk clears the undo stack.
        old_body = p.b
        old_p = p.copy()
        results = c.beautify_with_ruff(contents, root, filename)

        # Always suppress the reload prompt.
        efc.set_time(filename)

        # Do nothing else if there has been no change.
        if contents == results:
            return True

        if 0:  # Debugging only.
            sfn = g.shortFileName(filename)
            diff = difflib.unified_diff(
                g.splitLines(contents),
                g.splitLines(results),
                fromfile=f"Old: {sfn}",
                tofile=f"New {sfn}",
            )
            g.printObj(list(diff), tag=f"Diff: {sfn}")

        # Reload the file, suppressing the reload prompt.
        g.es(f"beautified: {g.shortFileName(filename)}", color='blue')
        c.refreshFromDisk(root)

        # Handle the undo *after* calling c.refreshFromDisk.
        bunch = u.beforeChangeBody(old_p)
        bunch.oldBody = old_body
        u.afterChangeBody(old_p, 'ruff-format', bunch)

        # #4159: This may not restore the outline as it was,
        # but it's much better than doing nothing.
        c.selectPosition(old_p)
        c.contractAllOtherNodes()
        c.redraw(old_p)
        return True

    # @+node:ekr.20221128123139.1: *6* at.runFlake8
    def runFlake8(self, root: Position) -> bool:  # pragma: no cover
        """Run flake8 on the selected node."""
        try:
            from leo.commands import checkerCommands

            if checkerCommands.flake8:
                x = checkerCommands.Flake8Command(self.c)
                x.run(root)
                return True
            return True  # Suppress error if pyflakes can not be imported.
        except Exception:
            g.es_exception()
            return True  # Pretend all is well

    # @+node:ekr.20161021084954.1: *6* at.runPyflakes
    def runPyflakes(self, root: Position) -> bool:  # pragma: no cover
        """Run pyflakes on the selected node."""
        try:
            from leo.commands import checkerCommands

            if checkerCommands.pyflakes:
                x = checkerCommands.PyflakesCommand(self.c)
                ok = x.run(root)
                return ok
            return True  # Suppress error if pyflakes can not be imported.
        except Exception:
            g.es_exception()
            return True  # Pretend all is well

    # @+node:ekr.20240924054208.1: *6* at.runPylint
    def runPylint(self, root: Position) -> bool:  # pragma: no cover
        """Run pylint on the selected node."""
        try:
            from leo.commands import checkerCommands

            if checkerCommands.lint:
                x = checkerCommands.PylintCommand(self.c)
                ok = bool(x.run(root))
                return ok
            return True  # Suppress error if pylint can not be imported.
        except Exception:
            g.es_exception()
            return True  # Pretend all is well

    # @+node:ekr.20041005105605.198: *5* at.directiveKind4 (write logic)
    # These patterns exclude constructs such as @encoding.setter or @encoding(whatever)
    # However, they must allow @language python, @nocolor-node, etc.

    at_directive_kind_pattern = re.compile(r'\s*@([\w-]+)\s*')

    def directiveKind4(self, s: str, i: int) -> int:
        """
        Return the kind of at-directive or noDirective.

        Potential simplifications:
        - Using strings instead of constants.
        - Using additional regex's to recognize directives.
        """
        at = self
        n = len(s)
        if i >= n or s[i] != '@':
            j = g.skip_ws(s, i)
            if g.match_word(s, j, "@others"):
                return at.othersDirective
            if g.match_word(s, j, "@all"):
                return at.allDirective
            return at.noDirective
        table = (
            ("@all", at.allDirective),
            ("@c", at.cDirective),
            ("@code", at.codeDirective),
            ("@doc", at.docDirective),
            ("@others", at.othersDirective),
            ("@verbatim", at.startVerbatim),
        )
        # ("@end_raw", at.endRawDirective),  # #2276.
        # ("@raw", at.rawDirective),  # #2276
        # Rewritten 6/8/2005.
        if i + 1 >= n or s[i + 1] in (' ', '\t', '\n'):
            # Bare '@' not recognized in cweb mode.
            return at.noDirective if at.language == "cweb" else at.atDirective
        if not s[i + 1].isalpha():
            return at.noDirective  # Bug fix: do NOT return miscDirective here!
        if at.language == "cweb" and g.match_word(s, i, '@c'):
            return at.noDirective
        # When the language is elixir, @doc followed by a space and string delimiter
        # needs to be treated as plain text; the following does not enforce the
        # 'string delimiter' part of that.  An @doc followed by something other than
        # a space will fall through to usual Leo @doc processing.
        if at.language == "elixir" and g.match_word(s, i, '@doc '):  # pragma: no cover
            return at.noDirective
        for name, directive in table:
            if g.match_word(s, i, name):
                return directive
        # Support for add_directives plugin.
        # Use regex to properly distinguish between Leo directives
        # and python decorators.
        s2 = s[i:]
        if m := self.at_directive_kind_pattern.match(s2):
            word = m.group(1)
            if word not in g.globalDirectiveList:
                return at.noDirective
            s3 = s2[m.end(1) :]
            if s3 and s3[0] in ".(":
                return at.noDirective
            return at.miscDirective
        # An unusual case.
        return at.noDirective  # pragma: no cover

    # @+node:ekr.20041005105605.200: *5* at.isSectionName
    # returns (flag, end). end is the index of the character after the section name.

    def isSectionName(self, s: str, i: int) -> tuple[bool, int]:  # pragma: no cover
        at = self
        # Allow leading periods.
        while i < len(s) and s[i] == '.':
            i += 1
        if not g.match(s, i, at.section_delim1):
            return False, -1
        i = g.find_on_line(s, i, at.section_delim2)
        if i > -1:
            return True, i + len(at.section_delim2)
        return False, -1

    # @+node:ekr.20190111112442.1: *5* at.isWritable
    def isWritable(self, path: str) -> bool:  # pragma: no cover
        """Return True if the path is writable."""
        try:
            # os.access() may not exist on all platforms.
            ok = os.access(path, os.W_OK)
        except AttributeError:
            return True
        if not ok:
            g.es('read only:', repr(path), color='red')
        return ok

    # @+node:ekr.20041005105605.201: *5* at.os and allies
    # @+node:ekr.20041005105605.202: *6* at.oblank, oblanks & otabs
    def oblank(self) -> None:
        self.os(' ')

    def oblanks(self, n: int) -> None:  # pragma: no cover
        self.os(' ' * abs(n))

    def otabs(self, n: int) -> None:  # pragma: no cover
        self.os('\t' * abs(n))

    # @+node:ekr.20041005105605.203: *6* at.onl & onl_sent
    def onl(self) -> None:
        """Write a newline to the output stream."""
        self.os('\n')  # **not** self.output_newline

    def onl_sent(self) -> None:
        """Write a newline to the output stream, provided we are outputting sentinels."""
        if self.sentinels:
            self.onl()

    # @+node:ekr.20041005105605.204: *6* at.os
    def os(self, s: str) -> None:
        """
        Append a string to at.outputList.

        All output produced by leoAtFile module goes here.
        """
        at = self
        s = g.toUnicode(s, at.encoding)
        at.outputList.append(s)

    # @+node:ekr.20041005105605.205: *5* at.outputStringWithLineEndings
    def outputStringWithLineEndings(self, s: str) -> None:  # pragma: no cover
        """
        Write the string s as-is except that we replace '\n' with the proper line ending.

        Calling self.onl() runs afoul of queued newlines.
        """
        at = self
        s = g.toUnicode(s, at.encoding)
        s = s.replace('\n', at.output_newline)
        self.os(s)

    # @+node:ekr.20190111045822.1: *5* at.precheck (calls shouldPrompt...)
    def precheck(self, fileName: str, root: Position) -> bool:  # pragma: no cover
        """
        Check whether a dirty, potentially dangerous, file should be written.

        Return True if so.  Return False *and* issue a warning otherwise.
        """
        at = self
        if g.unitTesting:
            return True
        # #1450: First, check that the directory exists.
        theDir = g.os_path_dirname(fileName)
        if theDir and not g.os_path_exists(theDir):
            at.error(f"{root.h}: Directory not found:\n{theDir}")
            return False
        # Now check the file.
        if not at.shouldPromptForDangerousWrite(fileName, root):
            # Fix bug 889175: Remember the full fileName.
            at.rememberReadPath(fileName, root)
            return True
        # Prompt if the write would overwrite the existing file.
        ok = self.promptForDangerousWrite(fileName)
        if ok:
            # Fix bug 889175: Remember the full fileName.
            at.rememberReadPath(fileName, root)
            return True
        # #1031: do not add @ignore here!
        g.es("not written:", fileName)
        return False

    # @+node:ekr.20050506090446.1: *5* at.putAtFirstLines
    def putAtFirstLines(self, s: str) -> None:
        """
        Write any @firstlines from string s.
        These lines are converted to @verbatim lines,
        so the read logic simply ignores lines preceding the @+leo sentinel.
        """
        at = self
        tag = "@first"
        i = 0
        while g.match(s, i, tag):
            i += len(tag)
            i = g.skip_ws(s, i)
            j = i
            i = g.skip_to_end_of_line(s, i)
            # Write @first line, whether empty or not
            line = s[j:i]
            at.os(line)
            at.onl()
            i = g.skip_nl(s, i)

    # @+node:ekr.20050506090955: *5* at.putAtLastLines
    def putAtLastLines(self, s: str) -> None:
        """
        Write any @last lines from string s.
        These lines are converted to @verbatim lines,
        so the read logic simply ignores lines following the @-leo sentinel.
        """
        at = self
        tag = "@last"
        # Use g.splitLines to preserve trailing newlines.
        lines = g.splitLines(s)
        n = len(lines)
        j = k = n - 1
        # Scan backwards for @last directives.
        while j >= 0:
            line = lines[j]
            if g.match(line, 0, tag):
                j -= 1
            elif not line.strip():
                j -= 1
            else:
                break  # pragma: no cover (coverage bug)
        # Write the @last lines.
        for line in lines[j + 1 : k + 1]:
            if g.match(line, 0, tag):
                i = len(tag)
                i = g.skip_ws(line, i)
                at.os(line[i:])

    # @+node:ekr.20041005105605.206: *5* at.putDirective & helper
    def putDirective(self, s: str, i: int, p: Position) -> int:
        r"""
        Output a sentinel a directive or reference s.

        It is important for PHP and other situations that \@first and \@last
        directives get translated to verbatim lines that do *not* include what
        follows the @first & @last directives.
        """
        at = self
        k = i
        j = g.skip_to_end_of_line(s, i)
        directive = s[i:j]
        if g.match_word(s, k, "@delims"):
            at.putDelims(directive, s, k)
        elif g.match_word(s, k, "@language"):
            self.putSentinel("@" + directive)
        elif g.match_word(s, k, "@comment"):
            self.putSentinel("@" + directive)
        elif g.match_word(s, k, "@last"):
            # #1307.
            if p.isAtCleanNode():  # pragma: no cover
                at.error(f"ignoring @last directive in {p.h!r}")
                g.es_print('@last is not valid in @clean nodes')
            # #1297.
            elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
                # Convert to an verbatim line _without_ anything else.
                self.putSentinel("@@last")
            else:
                at.error(f"ignoring @last directive in {p.h!r}")  # pragma: no cover
        elif g.match_word(s, k, "@first"):
            # #1307.
            if p.isAtCleanNode():  # pragma: no cover
                at.error(f"ignoring @first directive in {p.h!r}")
                g.es_print('@first is not valid in @clean nodes')
            # #1297.
            elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
                # Convert to an verbatim line _without_ anything else.
                self.putSentinel("@@first")
            else:
                at.error(f"ignoring @first directive in {p.h!r}")  # pragma: no cover
        else:
            self.putSentinel("@" + directive)
        i = g.skip_line(s, k)
        return i

    # @+node:ekr.20041005105605.207: *6* at.putDelims
    def putDelims(self, directive: str, s: str, k: int) -> None:
        """Put an @delims directive."""
        at = self
        # Put a space to protect the last delim.
        at.putSentinel(directive + " ")  # 10/23/02: put @delims, not @@delims
        # Skip the keyword and whitespace.
        j = i = g.skip_ws(s, k + len("@delims"))
        # Get the first delim.
        while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
            i += 1
        if j < i:
            at.startSentinelComment = s[j:i]
            # Get the optional second delim.
            j = i = g.skip_ws(s, i)
            while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
                i += 1
            at.endSentinelComment = s[j:i] if j < i else ""
        else:
            at.writeError("Bad @delims directive")  # pragma: no cover

    # @+node:ekr.20041005105605.210: *5* at.putIndent
    def putIndent(self, n: int, s: str = '') -> None:  # pragma: no cover
        """Put tabs and spaces corresponding to n spaces,
        assuming that we are at the start of a line.
        """
        if n > 0:
            w = self.tab_width
            if w > 1:
                q, r = divmod(n, w)
                self.otabs(q)
                self.oblanks(r)
            else:
                self.oblanks(n)

    # @+node:ekr.20041005105605.211: *5* at.putInitialComment
    def putInitialComment(self) -> None:  # pragma: no cover
        c = self.c
        s2 = c.config.getString('output-initial-comment') or ''
        if s2.strip():
            lines = s2.split("\\n")
            for line in lines:
                line = line.replace("@date", time.asctime())
                if line:
                    self.putSentinel("@comment " + line)

    # @+node:ekr.20190111172114.1: *5* at.replaceFile & helpers
    def replaceFile(
        self,
        contents: str,
        encoding: str,
        fileName: str,
        root: Position,
        ignoreBlankLines: bool = False,
    ) -> bool:
        """
        Write or create the given file from the contents.
        Return True if the original file was changed.
        """
        at, c = self, self.c
        assert root, g.callers()
        root.clearDirty()

        # Create the timestamp (only for messages).
        if c.config.getBool('log-show-save-time', default=False):  # pragma: no cover
            format = c.config.getString('log-timestamp-format') or "%H:%M:%S"
            timestamp = time.strftime(format) + ' '
        else:
            timestamp = ''

        # Adjust the contents.
        assert isinstance(contents, str), g.callers()
        if at.output_newline != '\n':  # pragma: no cover
            contents = contents.replace('\r', '').replace('\n', at.output_newline)

        # If file does not exist, create it from the contents.
        fileName = g.os_path_realpath(fileName)
        sfn = g.shortFileName(fileName)
        if not g.os_path_exists(fileName):
            ok = g.writeFile(contents, encoding, fileName)
            if ok:
                c.setFileTimeStamp(fileName)
                if not g.unitTesting:
                    g.es(f"{timestamp}created: {fileName}")  # pragma: no cover
                if root:
                    # Fix bug 889175: Remember the full fileName.
                    at.rememberReadPath(fileName, root)
                    at.checkPythonCode(contents, fileName, root)
            else:
                at.addToOrphanList(root)  # pragma: no cover
            # No original file to change. Return value tested by a unit test.
            return False  # No change to original file.

        old_contents = g.readFileIntoUnicodeString(fileName, encoding=at.encoding, silent=True)
        if not old_contents:
            old_contents = ''

        unchanged = (
            contents == old_contents
            or (not at.explicitLineEnding and at.compareIgnoringLineEndings(old_contents, contents))
            or ignoreBlankLines
            and at.compareIgnoringBlankLines(old_contents, contents)
        )

        if unchanged:
            at.unchangedFiles += 1
            if not g.unitTesting and c.config.getBool('report-unchanged-files', default=True):
                g.es(f"{timestamp}unchanged: {sfn}")  # pragma: no cover

            # Check *unchanged* files.
            at.checkPythonCode(contents, fileName, root)
            return False  # No change to original file.

        # Warn if we are only adjusting the line endings.
        if at.explicitLineEnding:  # pragma: no cover
            ok = (
                at.compareIgnoringLineEndings(old_contents, contents)
                or ignoreBlankLines
                and at.compareIgnoringLineEndings(old_contents, contents)
            )
            if not ok:
                g.warning("correcting line endings in:", fileName)

        # Write a changed file.
        ok = g.writeFile(contents, encoding, fileName)
        if ok:
            c.setFileTimeStamp(fileName)
            if not g.unitTesting and c.config.getBool('report-changed-files', default=True):
                g.es(f"{timestamp}wrote: {sfn}")  # pragma: no cover
        else:  # pragma: no cover
            g.error('error writing', sfn)
            g.es('not written:', sfn)
            at.addToOrphanList(root)

        # Check *after* writing the file.
        at.checkPythonCode(contents, fileName, root)
        return ok

    # @+node:ekr.20190114061452.27: *6* at.compareIgnoringBlankLines
    def compareIgnoringBlankLines(self, s1: str, s2: str) -> bool:  # pragma: no cover
        """Compare two strings, ignoring blank lines."""
        if s1 == s2:
            return True
        s1 = g.removeBlankLines(s1)
        s2 = g.removeBlankLines(s2)
        return s1 == s2

    # @+node:ekr.20190114061452.28: *6* at.compareIgnoringLineEndings
    def compareIgnoringLineEndings(self, s1: str, s2: str) -> bool:  # pragma: no cover
        """Compare two strings, ignoring line endings."""
        if s1 == s2:
            return True
        # Wrong: equivalent to ignoreBlankLines!
        # s1 = s1.replace('\n','').replace('\r','')
        # s2 = s2.replace('\n','').replace('\r','')
        s1 = s1.replace('\r', '')
        s2 = s2.replace('\r', '')
        return s1 == s2

    # @+node:ekr.20211029052041.1: *5* at.scanRootForSectionDelims
    def scanRootForSectionDelims(self, root: Position) -> None:
        """
        Scan root.b for an "@section-delims" directive.
        Set section_delim1 and section_delim2 ivars.
        """
        at = self
        # Set defaults.
        at.section_delim1 = '<<'
        at.section_delim2 = '>>'
        # Scan root.b.
        lines = []
        for s in g.splitLines(root.b):
            if m := g.g_section_delims_pat.match(s):
                lines.append(s)
                at.section_delim1 = m.group(1)
                at.section_delim2 = m.group(2)
        # Disallow multiple directives.
        if len(lines) > 1:  # pragma: no cover
            at.error(f"Multiple @section-delims directives in {root.h}")
            g.es_print('using default delims')
            at.section_delim1 = '<<'
            at.section_delim2 = '>>'

    # @+node:ekr.20090514111518.5665: *5* at.tabNannyNode
    def tabNannyNode(self, p: Position, body: str) -> None:
        try:
            readline = g.ReadLinesClass(body).next
            tabnanny.process_tokens(tokenize.generate_tokens(readline))
        except IndentationError:  # pragma: no cover
            if g.unitTesting:
                raise
            junk2, msg, junk = sys.exc_info()
            g.error("IndentationError in", p.h)
            g.es('', str(msg))
        except tokenize.TokenError:  # pragma: no cover
            if g.unitTesting:
                raise
            junk3, msg, junk = sys.exc_info()
            g.error("TokenError in", p.h)
            g.es('', str(msg))
        except tabnanny.NannyNag:  # pragma: no cover
            if g.unitTesting:
                raise
            junk4, nag, junk = sys.exc_info()
            badline = nag.get_lineno()
            line = nag.get_line()
            message = nag.get_msg()
            g.error("indentation error in", p.h, "line", badline)
            g.es(message)
            line2 = repr(str(line))[1:-1]
            g.es("offending line:\n", line2)
        except Exception:  # pragma: no cover
            g.trace("unexpected exception")
            g.es_exception()
            raise

    # @+node:ekr.20041005105605.216: *5* at.warnAboutOrpanAndIgnoredNodes
    # Called from putFile.

    def warnAboutOrphandAndIgnoredNodes(self) -> None:  # pragma: no cover
        # Always warn, even when language=="cweb"
        at, root = self, self.root
        message = 'The node is included neither by an @others directive nor a section reference'
        if at.errors:
            return  # No need to repeat this.
        if root.isAtAllNode():  # #4729.
            return
        if root.isOrphan():  # #4729.
            return
        for p in root.subtree(copy=False):
            if not p.v.isVisited():
                at.writeError("Orphan node:  " + p.h)
                g.es_print_unique_message(message)
                if p.hasParent():
                    g.blue("parent node:", p.parent().h)

    # @+node:ekr.20041005105605.217: *5* at.writeError
    def writeError(self, message: str) -> None:  # pragma: no cover
        """Issue an error while writing an @<file> node."""
        at = self
        if at.errors == 0:
            fn = at.targetFileName or 'unnamed file'
            g.es_error(f"errors writing: {fn}")
        at.error(message)
        at.addToOrphanList(at.root)

    # @+node:ekr.20041005105605.218: *5* at.writeException
    def writeException(self, fileName: str, root: Position) -> None:  # pragma: no cover
        at = self
        g.error("exception writing:", fileName)
        g.es_exception()
        if getattr(at, 'outputFile', None):
            at.outputFile.flush()
            at.outputFile.close()
            at.outputFile = None
        at.remove(fileName)
        at.addToOrphanList(root)

    # @+node:ekr.20041005105605.219: *3* at.Utilities
    # @+node:ekr.20041005105605.220: *4* at.error & printError
    def error(self, *args: Args) -> None:  # pragma: no cover
        at = self
        at.printError(*args)
        at.errors += 1

    def printError(self, *args: Args) -> None:  # pragma: no cover
        """Print an error message that may contain non-ascii characters."""
        at = self
        if at.errors:
            g.error(*args)
        else:
            g.warning(*args)

    # @+node:ekr.20041005105605.221: *4* at.exception
    def exception(self, message: str) -> None:  # pragma: no cover
        self.error(message)
        g.es_exception()

    # @+node:ekr.20050104131929: *4* at.file operations...
    # Error checking versions of corresponding functions in Python's os module.
    # @+node:ekr.20050104131820: *5* at.chmod
    def chmod(self, fileName: str, mode: int) -> None:  # pragma: no cover
        # Do _not_ call self.error here.
        if mode is None:
            return
        try:
            os.chmod(fileName, mode)
        except Exception:
            g.es("exception in os.chmod", fileName)
            g.es_exception()

    # @+node:ekr.20050104132018: *5* at.remove
    def remove(self, fileName: str) -> bool:  # pragma: no cover
        if not fileName:
            g.trace('No file name', g.callers())
            return False
        try:
            os.remove(fileName)
            return True
        except Exception:
            if not g.unitTesting:
                self.error(f"exception removing: {fileName}")
                g.es_exception()
            return False

    # @+node:ekr.20050104132026: *5* at.stat
    def stat(self, fileName: str) -> int:  # pragma: no cover
        """Return the access mode of named file, removing any setuid, setgid, and sticky bits."""
        # Do _not_ call self.error here.
        try:
            mode = (os.stat(fileName))[0] & (7 * 8 * 8 + 7 * 8 + 7)  # 0777
        except Exception:
            mode = None
        return mode

    # @+node:ekr.20090530055015.6023: *4* at.get/setPathUa
    def getPathUa(self, p: Position) -> str:
        if hasattr(p.v, 'tempAttributes'):
            d = p.v.tempAttributes.get('read-path', {})
            return d.get('path')
        return ''

    def setPathUa(self, p: Position, path: str) -> None:
        if not hasattr(p.v, 'tempAttributes'):
            p.v.tempAttributes = {}
        d = p.v.tempAttributes.get('read-path', {})
        d['path'] = path
        p.v.tempAttributes['read-path'] = d

    # @+node:ekr.20090712050729.6017: *4* at.promptForDangerousWrite
    def promptForDangerousWrite(
        self, fileName: str, message: str = None
    ) -> bool:  # pragma: no cover
        """Raise a dialog asking the user whether to overwrite an existing file."""
        at, c, root = self, self.c, self.root
        if at.cancelFlag:
            assert at.canCancelFlag
            return False
        if at.yesToAll:
            assert at.canCancelFlag
            return True
        if root and root.h.startswith('@auto-rst'):
            # Fix bug 50: body text lost switching @file to @auto-rst
            # Refuse to convert any @<file> node to @auto-rst.
            d = root.v.at_read if hasattr(root.v, 'at_read') else {}
            aList = sorted(d.get(fileName, []))
            for h in aList:
                if not h.startswith('@auto-rst'):
                    g.es('can not convert @file to @auto-rst!', color='red')
                    g.es('reverting to:', h)
                    root.h = h
                    c.redraw()
                    return False
        if message is None:
            message = (
                f"{g.splitLongFileName(fileName)}\n"
                f"{g.tr('already exists.')}\n"
                f"{g.tr('Overwrite this file?')}"
            )  # fmt: skip
        result = g.app.gui.runAskYesNoCancelDialog(
            c,
            title='Overwrite existing file?',
            yesToAllMessage="Yes To &All",
            message=message,
            cancelMessage="&Cancel (No To All)",
        )
        if at.canCancelFlag:
            # We are in the writeAll logic so these flags can be set.
            if result == 'cancel':
                at.cancelFlag = True
            elif result == 'yes-to-all':
                at.yesToAll = True
        return result in ('yes', 'yes-to-all')

    # @+node:ekr.20120112084820.10001: *4* at.rememberReadPath
    def rememberReadPath(self, fn: str, p: Position) -> None:
        """
        Remember the files that have been read *and*
        the full headline (@<file> type) that caused the read.
        """
        v = p.v
        # #50: body text lost switching @file to @auto-rst
        if not hasattr(v, 'at_read'):
            v.at_read = {}  # pragma: no cover
        d = v.at_read
        aSet = d.get(fn, set())
        aSet.add(p.h)
        d[fn] = aSet

    # @+node:ekr.20231207062321.1: *4* at.repr_kind
    def repr_kind(self, kind: int) -> str:  # pragma: no cover (testing)
        """Return the string representation of AtFile class constants."""
        d: dict[int, str] = {
            1: 'plain',
            2: '@all',
            3: '@doc',
            4: '@ ',
            5: '@code',
            6: '@c',
            7: '@others',
            8: '@<misc>',
            9: '@verbatim',
        }
        return d.get(kind) or f"<unknown AtFile class constant> {kind!r}"

    # @+node:ekr.20120110174009.9965: *4* at.shouldPromptForDangerousWrite
    def shouldPromptForDangerousWrite(self, fn: str, p: Position) -> bool:  # pragma: no cover
        """
        Return True if Leo should warn the user that p is an @<file> node that
        was not read during startup. Writing that file might cause data loss.

        See #50: https://github.com/leo-editor/leo-editor/issues/50
        """
        at = self
        efc = g.app.externalFilesController

        if p.isAtNoSentFileNode():
            # No danger of overwriting a file: It was never read.
            return False

        if not g.os_path_exists(fn):
            # No danger of overwriting fn.
            return False

        # #4385: Honor yes-to-all or no-to-all.
        if at.cancelFlag or at.yesToAll:
            return False

        # Prompt if the external file is newer.
        if efc and efc.has_changed(fn):
            g.trace('Changed:', fn)
            return True

        if p.isAtCleanNode():  # #4385.
            return False

        if hasattr(p.v, 'at_read'):
            # #50: body text lost switching @file to @auto-rst
            d = p.v.at_read
            for k in d:
                # Make sure k still exists.
                if os.path.exists(k) and os.path.samefile(k, fn) and p.h in d.get(k, set()):
                    d[fn] = d[k]
                    return False
            aSet = d.get(fn, set())
            return p.h not in aSet

        return True  # The file was never read.

    # @+node:ekr.20041005105605.20: *4* at.warnOnReadOnlyFile
    def warnOnReadOnlyFile(self, fn: str) -> None:
        # os.access() may not exist on all platforms.
        try:
            read_only = not os.access(fn, os.W_OK)
        except AttributeError:  # pragma: no cover
            read_only = False
        if read_only:
            g.error("read only:", fn)  # pragma: no cover

    # @-others


atFile = AtFile  # compatibility


# @+node:ekr.20180602102448.1: ** class FastAtRead
class FastAtRead:
    """
    Read an external file, created from an @file tree.
    This is Vitalije's code, edited by EKR.
    """

    # @+others
    # @+node:ekr.20211030193146.1: *3* fast_at.__init__
    def __init__(self, c: Cmdr, gnx2vnode: dict[str, VNode]) -> None:
        self.c = c
        assert gnx2vnode is not None
        # The global fc.gnxDict. Keys are gnx's, values are vnodes.
        self.gnx2vnode: dict[str, VNode] = gnx2vnode
        self.path: str = None
        self.root: Position = None
        # compiled patterns...
        self.after_pat: re.Pattern = None
        self.all_pat: re.Pattern = None
        self.code_pat: re.Pattern = None
        self.comment_pat: re.Pattern = None
        self.delims_pat: re.Pattern = None
        self.doc_pat: re.Pattern = None
        self.first_pat: re.Pattern = None
        self.last_pat: re.Pattern = None
        self.node_start_pat: re.Pattern = None
        self.others_pat: re.Pattern = None
        self.ref_pat: re.Pattern = None
        self.section_delims_pat: re.Pattern = None

    # @+node:ekr.20180602103135.3: *3* fast_at.get_patterns
    def get_patterns(self, comment_delims: tuple) -> None:
        """Create regex patterns for the given comment delims."""

        # This must be a function, because of @comments & @delims.
        comment_delim_start, comment_delim_end = comment_delims

        # Make no assumption about comment delims.
        delim1 = re.escape(comment_delim_start)
        delim2 = re.escape(comment_delim_end or '')
        ref = g.angleBrackets(r'(.*)')

        # The @+leo sentinel MUST determine the format of *all* sentinel comments.
        # The patterns below MUST NOT accept a space after {delim1} !!!
        # Doing so will break Leo!
        table = (
            # These patterns must be mutually exclusive.
            ('after',       fr'^\s*{delim1}@afterref{delim2}$'),              # @afterref
            ('all',         fr'^(\s*){delim1}@(\+|-)all\b(.*){delim2}$'),     # @all
            ('code',        fr'^\s*{delim1}@@c(ode)?\b(.*){delim2}$'),        # @c and @code
            ('comment',     fr'^\s*{delim1}@@comment(.*){delim2}'),           # @comment
            ('delims',      fr'^\s*{delim1}@delims(.*){delim2}'),             # @delims
            ('doc',         fr'^\s*{delim1}@\+(at|doc)?(\s.*?)?{delim2}\n'),  # @doc or @
            ('first',       fr'^\s*{delim1}@@first{delim2}$'),                # @first
            ('last',        fr'^\s*{delim1}@@last{delim2}$'),                 # @last
            # @verbatim
            # @node
            ('node_start',  fr'^(\s*){delim1}@\+node:([^:]+): \*(\d+)?(\*?) (.*){delim2}$'),
            ('others',      fr'^(\s*){delim1}@(\+|-)others\b(.*){delim2}$'),  # @others
            ('ref',         fr'^(\s*){delim1}@(\+|-){ref}\s*{delim2}$'),      # section ref
            # @verbatim
            # @section-delims
            ('section_delims', fr'^\s*{delim1}@@section-delims[ \t]+([^ \w\n\t]+)[ \t]+([^ \w\n\t]+)[ \t]*{delim2}$'),
        )  # fmt: skip

        # Set the ivars.
        self.pattern_ivars: list[str] = []  # For debugging only.
        for name, pattern in table:
            ivar = f"{name}_pat"
            self.pattern_ivars.append(ivar)
            assert hasattr(self, ivar), ivar
            setattr(self, ivar, re.compile(pattern))

    # @+node:ekr.20180602103135.2: *3* fast_at.scan_header
    header_pattern = re.compile(
        r'''
        ^(.+)@\+leo
        (-ver=(\d+))?
        (-thin)?
        (-encoding=(.*)(\.))?
        (.*)$''',
        re.VERBOSE,
    )

    def scan_header(self, lines: list[str]) -> Optional[tuple[tuple, list[str], int]]:
        """
        Scan for the header line, which follows any @first lines.
        Return (delims, first_lines, i+1) or None

        *Note*: delims[0] will end with a blank when reading a file with blackened sentinels!
        """
        first_lines: list[str] = []
        i = 0  # To keep some versions of pylint happy.
        for i, line in enumerate(lines):
            if m := self.header_pattern.match(line):
                delims = m.group(1), m.group(8) or ''
                return delims, first_lines, i + 1
            first_lines.append(line)
        return None  # pragma: no cover (defensive)

    # @+node:ekr.20180602103135.8: *3* fast_at.scan_lines
    def scan_lines(
        self,
        comment_delims: tuple,
        first_lines: list[str],
        lines: list[str],
        path: str,
        start: int,
    ) -> None:
        """Scan all lines of the file, creating vnodes."""
        # @+<< init scan_lines >>
        # @+node:ekr.20180602103135.9: *4* << init scan_lines >>
        #
        # Simple vars...
        afterref = False  # True: the next line follows @afterref.
        clone_v: VNode = None  # The root of the clone tree.
        # The start/end *comment* delims.
        # Important: scan_header ends comment_delim1 with a blank when using black sentinels.
        comment_delim1, comment_delim2 = comment_delims
        doc_skip = (
            comment_delim1 + '\n',
            comment_delim2 + '\n',
        )  # To handle doc parts.
        first_i = 0  # Index into first array.
        in_doc = False  # True: in @doc parts.
        is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'  # True: cweb hack in effect.
        indent = 0  # The current indentation.
        level_stack: list[tuple[VNode, VNode]] = []
        n_last_lines = 0  # The number of @@last directives seen.

        # #1065 so reads will not create spurious child nodes.
        root_seen = False  # False: The next +@node sentinel denotes the root, regardless of gnx.
        section_delim1 = '<<'
        section_delim2 = '>>'
        section_reference_seen = False
        sentinel = comment_delim1 + '@'

        # at+others, at+<section>, and at+all update the stack.
        stack: list[tuple[str, int, list[str]]] = []  # Entries are (gnx, indent, body)
        verbatim_line = comment_delim1 + '@verbatim' + comment_delim2
        verbatim = False  # True: the next line must be added without change.

        # Init the parent vnode.
        gnx = self.root.gnx
        context = self.c
        parent_v = self.root.v
        root_v = parent_v  # Does not change.
        level_stack.append((root_v, None))

        # Init the gnx dict last.
        gnx2vnode = self.gnx2vnode  # Keys are gnx's, values are vnodes.
        gnx2body: dict[str, list[str]] = {}  # Keys are gnxs, values are list of body lines.
        gnx2vnode[gnx] = parent_v  # Add gnx to the keys

        # Add gnx to the keys.
        # Body is the list of lines presently being accumulated.
        gnx2body[gnx] = body = first_lines

        # Set the patterns
        self.get_patterns(comment_delims)
        # @-<< init scan_lines >>
        i = 0  # To keep pylint happy.
        for i, line in enumerate(lines[start:]):
            # @+<< trace patterns >>
            # @+node:ekr.20231205061620.1: *4* << trace patterns >>
            # Keep this excellent trace, but comment it out for production code.
            # if g.unitTesting:
            # for ivar in self.pattern_ivars:
            # pattern = getattr(self, ivar)
            # assert isinstance(pattern, re.Pattern)
            # m = pattern.match(line)
            # if m:
            # print(f"{i:2} {ivar:15} {line!r}")
            # break
            # else:
            # print(f"{i:2} {' ':15} {line!r}")
            # @-<< trace patterns >>
            # Strip the line only once.
            strip_line = line.strip()
            if afterref:
                # @+<< handle afterref line>>
                # @+node:ekr.20211102052251.1: *4* << handle afterref line >>
                if body:  # a list of lines.
                    body[-1] = body[-1].rstrip() + line
                else:
                    body = [line]  # pragma: no cover
                afterref = False
                # @-<< handle afterref line>>
                continue
            if verbatim:
                # @+<< handle verbatim line >>
                # @+node:ekr.20211102052518.1: *4* << handle verbatim line >>
                # Previous line was verbatim *sentinel*. Append this line as it is.

                # 2022/12/02: Bug fix: adjust indentation.
                if indent and line[:indent].isspace() and len(line) > indent:
                    line = line[indent:]  # pragma: no cover (defensive?)

                body.append(line)
                verbatim = False
                # @-<< handle verbatim line >>
                continue
            if strip_line == verbatim_line:
                verbatim = True
                continue
            # @+<< finalize line >>
            # @+node:ekr.20180602103135.10: *4* << finalize line >>
            # Undo the cweb hack.
            if is_cweb and line.startswith(sentinel):
                line = line[: len(sentinel)] + line[len(sentinel) :].replace('@@', '@')
            # Adjust indentation.
            if indent and line[:indent].isspace() and len(line) > indent:
                line = line[indent:]
            # @-<< finalize line >>
            # Faster than a regex!
            if not in_doc and not strip_line.startswith(sentinel):
                body.append(line)
                continue
            # These three sections might clear in_doc.
            # @+<< handle @others >>
            # @+node:ekr.20180602103135.14: *4* << handle @others >>
            if m := self.others_pat.match(line):
                in_doc = False
                if m.group(2) == '+':  # opening sentinel
                    body.append(f"{m.group(1)}@others{m.group(3) or ''}\n")
                    stack.append((gnx, indent, body))
                    indent += m.end(1)  # adjust current indentation
                else:  # closing sentinel.
                    # m.group(2) is '-' because the pattern matched.
                    try:
                        gnx, indent, body = stack.pop()
                    except IndexError:  # pragma: no cover
                        # #2624: This can happen during git-diff.
                        pass
                continue
            # @-<< handle @others >>
            # @+<< handle section refs >>
            # @+node:ekr.20180602103135.18: *4* << handle section refs >>
            # Note: scan_header sets *comment* delims, not *section* delims.
            # This section coordinates with the section that handles @section-delims.
            if m := self.ref_pat.match(line):
                in_doc = False
                if m.group(2) == '+':
                    # Any later @section-delims directive is a serious error.
                    # This kind of error should have been caught by Leo's atFile write logic.
                    section_reference_seen = True
                    # open sentinel.
                    body.append(m.group(1) + section_delim1 + m.group(3) + section_delim2 + '\n')
                    stack.append((gnx, indent, body))
                    indent += m.end(1)
                elif stack:
                    # m.group(2) is '-' because the pattern matched.
                    gnx, indent, body = stack.pop()  # #1232: Only if the stack exists.
                continue  # 2021/10/29: *always* continue.
            # @-<< handle section refs >>
            # @+<< handle node_start >>
            # @+node:ekr.20180602103135.19: *4* << handle node_start >>
            if m := self.node_start_pat.match(line):
                in_doc = False
                gnx, head = m.group(2), m.group(5)
                # m.group(3) is the level number, m.group(4) is the number of stars.
                level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
                v = gnx2vnode.get(gnx)

                # Case 1: The root @file node. Don't change the headline.
                #         #3931: Always use root_v, but use the gnx from external file!
                if not root_seen:
                    root_seen = True
                    clone_v = None
                    v = root_v
                    if root_v.gnx != gnx:
                        # Delete all traces of root_v.gnx.
                        if root_v.gnx in gnx2body:
                            del gnx2body[root_v.gnx]
                        if root_v.gnx in gnx2vnode:
                            del gnx2vnode[root_v.gnx]
                        # `refresh-from-disk` issues this messages, but 'git-diff' should not.
                        # g.trace(f"Changing gnx! old: {root_v.gnx} new: {gnx} in {head}")
                        root_v.fileIndex = gnx
                    gnx2vnode[gnx] = root_v
                    gnx2body[gnx] = body = []
                    v.children = []
                    continue  # End of case 1.

                # Case 2: We are scanning the descendants of a clone.
                parent_v, clone_v = level_stack[level - 2]
                if v and clone_v:
                    # The last version of the body and headline wins..
                    gnx2body[gnx] = body = []
                    v._headString = head
                    # Update the level_stack.
                    level_stack = level_stack[: level - 1]
                    level_stack.append((v, clone_v))
                    # Always clear the children!
                    v.children = []
                    parent_v.children.append(v)
                    continue  # End of case 2.

                # Case 3: we are not already scanning the descendants of a clone.
                if v:
                    # The *start* of a clone tree. Reset the children.
                    clone_v = v
                    v.children = []
                else:
                    # Make a new vnode.
                    v = leoNodes.VNode(context=context, gnx=gnx)
                # The last version of the body and headline wins.
                gnx2vnode[gnx] = v
                gnx2body[gnx] = body = []
                v._headString = head
                # Update the stack.
                level_stack = level_stack[: level - 1]
                level_stack.append((v, clone_v))
                # Update the links.
                assert v != root_v
                parent_v.children.append(v)
                v.parents.append(parent_v)
                continue
            # @-<< handle node_start >>
            if in_doc:
                # @+<< handle @c or @code >>
                # @+node:ekr.20211031033532.1: *4* << handle @c or @code >>
                # When delim_end exists the doc block:
                # - begins with the opening delim, alone on its own line
                # - ends with the closing delim, alone on its own line.
                # Both of these lines should be skipped.
                #
                # #1496: Retire the @doc convention.
                #        An empty line is no longer a sentinel.

                assert in_doc, repr(line)

                if comment_delim2 and line in doc_skip:
                    # skip start/end comment delim!
                    continue
                #
                # Check for @c or @code.
                if m := self.code_pat.match(line):
                    in_doc = False
                    body.append('@code\n' if m.group(1) else '@c\n')
                    continue
                # @-<< handle @c or @code >>
            else:
                # @+<< handle @ or @doc >>
                # @+node:ekr.20211031033754.1: *4* << handle @ or @doc >>
                assert not in_doc, repr(line)

                if m := self.doc_pat.match(line):
                    # @verbatim
                    # @+at or @+doc?
                    doc = '@doc' if m.group(1) == 'doc' else '@'
                    doc2 = m.group(2) or ''  # Trailing text.
                    if doc2:
                        body.append(f"{doc}{doc2}\n")
                    else:
                        body.append(doc + '\n')
                    # Enter @doc mode.
                    in_doc = True
                    continue
                # @-<< handle @ or @doc >>
            if line.startswith(comment_delim1 + '@-leo'):  # Faster than a regex!
                # The @-leo sentinel adds *nothing* to the text.
                i += 1
                break
            # Order doesn't matter.
            # @+<< handle @all >>
            # @+node:ekr.20180602103135.13: *4* << handle @all >>
            if m := self.all_pat.match(line):
                # @verbatim
                # @all tells Leo's *write* code not to check for undefined sections.
                # Here, in the read code, we merely need to add it to the body.
                # Pushing and popping the stack may not be necessary, but it can't hurt.
                if m.group(2) == '+':  # opening sentinel
                    body.append(f"{m.group(1)}@all{m.group(3) or ''}\n")
                    stack.append((gnx, indent, body))
                else:  # closing sentinel.
                    # m.group(2) is '-' because the pattern matched.
                    gnx, indent, body = stack.pop()
                    gnx2body[gnx] = body
                continue
            # @-<< handle @all >>
            # @+<< handle afterref >>
            # @+node:ekr.20180603063102.1: *4* << handle afterref >>
            if m := self.after_pat.match(line):
                afterref = True
                continue
            # @-<< handle afterref >>
            # @+<< handle @first and @last >>
            # @+node:ekr.20180606053919.1: *4* << handle @first and @last >>
            if m := self.first_pat.match(line):
                if 0 <= first_i < len(first_lines):
                    body.append('@first ' + first_lines[first_i])
                    first_i += 1
                else:  # pragma: no cover
                    g.trace(f"\ntoo many @first lines: {path}")
                    print('@first is valid only at the start of @<file> nodes\n')
                    g.printObj(first_lines, tag='first_lines')
                    g.printObj(lines[start : i + 2], tag='lines[start:i+2]')
                continue

            if m := self.last_pat.match(line):
                # Just increment the count of the expected last lines.
                # We'll fill in the @last line directives after we see the @-leo directive.
                n_last_lines += 1
                continue
            # @-<< handle @first and @last >>
            # @+<< handle @comment >>
            # @+node:ekr.20180621050901.1: *4* << handle @comment >>
            # https://leo-editor.github.io/leo-editor/directives.html#part-4-dangerous-directives
            if m := self.comment_pat.match(line):
                # <1, 2 or 3 comment delims>
                delims = m.group(1).strip()

                # Whatever happens, retain the @delims line.
                body.append(f"@comment {delims}\n")
                delim1, delim2, delim3 = g.set_delims_from_string(delims)

                # delim1 is always the single-line delimiter.
                if delim1:
                    comment_delim1, comment_delim2 = delim1, ''
                else:
                    comment_delim1, comment_delim2 = (
                        delim2,
                        delim3,
                    )  # pragma: no cover (unimportant)

                # Within these delimiters:
                # - double underscores represent a newline.
                # - underscores represent a significant space,
                comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
                comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
                # Recalculate all delim-related values
                doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
                is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
                sentinel = comment_delim1 + '@'

                # Recalculate the patterns.
                comment_delims = comment_delim1, comment_delim2
                self.get_patterns(comment_delims)
                continue
            # @-<< handle @comment >>
            # @+<< handle @delims >>
            # @+node:ekr.20180608104836.1: *4* << handle @delims >>
            if m := self.delims_pat.match(line):
                # Get 1 or 2 comment delims
                # Whatever happens, retain the original @delims line.
                delims = m.group(1).strip()
                body.append(f"@delims {delims}\n")
                #
                # Parse the delims.
                self.delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
                m2 = self.delims_pat.match(delims)
                if not m2:  # pragma: no cover
                    g.trace(f"Ignoring invalid @delims: {line!r}")
                    continue
                comment_delim1 = m2.group(1)
                comment_delim2 = m2.group(2) or ''

                # Within these delimiters:
                # - double underscores represent a newline.
                # - underscores represent a significant space,
                comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
                comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')

                # Recalculate all delim-related values
                doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
                is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
                sentinel = comment_delim1 + '@'

                # Recalculate the patterns
                comment_delims = comment_delim1, comment_delim2
                self.get_patterns(comment_delims)
                continue
            # @-<< handle @delims >>
            # @+<< handle @section-delims >>
            # @+node:ekr.20211030033211.1: *4* << handle @section-delims >>
            if m := self.section_delims_pat.match(line):
                if section_reference_seen:  # pragma: no cover
                    # This is a serious error.
                    # This kind of error should have been caught by Leo's atFile write logic.
                    g.es_print('section-delims seen after a section reference', color='red')
                else:
                    # Carefully update the section reference pattern!
                    section_delim1 = d1 = re.escape(m.group(1))
                    section_delim2 = d2 = re.escape(m.group(2) or '')
                    self.ref_pat = re.compile(
                        rf'^(\s*){comment_delim1}@(\+|-){d1}(.*){d2}\s*{comment_delim2}$'
                    )
                body.append(f"@section-delims {m.group(1)} {m.group(2)}\n")
                continue
            # @-<< handle @section-delims >>
            # These sections must be last, in this order.
            # @+<< handle remaining @@ lines >>
            # @+node:ekr.20180603135602.1: *4* << handle remaining @@ lines >>
            # @verbatim
            # @first, @last, @delims and @comment generate @@ sentinels,
            # So this must follow all of those.
            if line.startswith(comment_delim1 + '@@'):
                ii = len(comment_delim1) + 1  # on second '@'
                jj = line.rfind(comment_delim2) if comment_delim2 else -1
                body.append(line[ii:jj] + '\n')
                continue
            # @-<< handle remaining @@ lines >>
            if in_doc:
                # @+<< handle remaining @doc lines >>
                # @+node:ekr.20180606054325.1: *4* << handle remaining @doc lines >>
                assert in_doc, repr(line)

                if comment_delim2:
                    # Inner doc lines have no comment delims.
                    body.append(line)
                    continue

                # Legacy:
                #    with --black-sentinels: comment_delim1 ends with a blank.
                # without --black-sentinels: comment_delim1 does *not* end with a blank.

                # Leo 6.7.6:
                #   comment_delim never ends with a blank.

                tail = line.lstrip()[len(comment_delim1.rstrip()) + 1 :]
                if tail.strip():
                    body.append(tail)
                else:
                    body.append('\n')
                continue
                # @-<< handle remaining @doc lines >>
            # @+<< handle remaining @ lines >>
            # @+node:ekr.20180602103135.17: *4* << handle remaining @ lines >>
            # Handle an apparent sentinel line.
            # This *can* happen after the git-diff or refresh-from-disk commands.
            #
            if 1:  # pragma: no cover (defensive)
                # This assert verifies the short-circuit test.
                assert strip_line.startswith(sentinel), repr(line)

                # Defensive: make *sure* we ignore verbatim lines.
                if strip_line == verbatim_line:
                    g.trace('Ignore bad @verbatim sentinel', repr(line))
                else:
                    # A useful trace.
                    g.trace(
                        f"{g.shortFileName(self.path)}: "
                        f"warning: inserting unexpected line: {line.rstrip()!r}"
                    )  # fmt: skip
                    # #2213: *Do* insert the line, with a warning.
                    body.append(line)
            # @-<< handle remaining @ lines >>
        else:
            # No @-leo sentinel!
            return  # pragma: no cover
        # @+<< final checks >>
        # @+node:ekr.20211104054823.1: *4* << final checks >>
        if g.unitTesting:
            assert not stack, stack
        elif stack:  # pragma: no cover
            g.error('scan_lines: Stack should be empty')
            g.printObj(stack, tag='stack')
        # @-<< final checks >>
        # @+<< insert @last lines >>
        # @+node:ekr.20211103101453.1: *4* << insert @last lines >>
        tail_lines = lines[start + i :]
        if tail_lines:
            # Convert the trailing lines to @last directives.
            last_lines = [f"@last {z.rstrip()}\n" for z in tail_lines]
            # Add the lines to the dictionary of lines.
            gnx2body[gnx] = gnx2body[gnx] + last_lines
            # Warn if there is an unexpected number of last lines.
            if n_last_lines != len(last_lines):  # pragma: no cover
                n1 = n_last_lines
                n2 = len(last_lines)
                g.trace(f"Expected {n1} trailing line{g.plural(n1)}, got {n2}")
        # @-<< insert @last lines >>
        # @+<< post pass: set all body text>>
        # @+node:ekr.20211104054426.1: *4* << post pass: set all body text>>
        # Set the body text.
        assert root_v.gnx in gnx2vnode, root_v
        assert root_v.gnx in gnx2body, root_v
        for key in gnx2body:
            body = gnx2body.get(key)
            v = gnx2vnode.get(key)
            assert v, (key, v)
            v._bodyString = g.toUnicode(''.join(body))
        # @-<< post pass: set all body text>>

    # @+node:ekr.20180603170614.1: *3* fast_at.read_into_root
    def read_into_root(self, contents: str, path: str, root: Position) -> bool:
        """
        Parse the file's contents, creating a tree of vnodes
        anchored in root.v.
        """
        self.path = path
        self.root = root
        contents = contents.replace('\r', '')
        lines = g.splitLines(contents)
        data = self.scan_header(lines)
        if not data:  # pragma: no cover
            g.trace(f"Invalid external file: {path}")
            return False
        # Clear all children.
        # Previously, this had been done in readOpenFile.
        root.v._deleteAllChildren()
        comment_delims, first_lines, start_i = data
        self.scan_lines(comment_delims, first_lines, lines, path, start_i)
        return True

    # @-others


# @+node:ekr.20240410111658.3: ** class LeoIOStatus
class LeoIOStatus:
    """A class representing the status of a Leo file operation"""

    at_comment_seen = False
    at_delims_seen = False
    at_warning_given = False
    has_at_others = False
    in_code = True

    def __repr__(self) -> str:  # pragma: no cover (testing)
        return (
            f"code? {int(self.in_code)} "
            # f"comment? {int(self.at_comment_seen)} "
            # f"delims? {int(self.at_delims_seen)} "
            # f"@others? {int(self.has_at_others)} "
        )


# @-others
# @@language python
# @@tabwidth -4
# @@pagewidth 60
# @-leo
