parser_document.go

package helium

import (
	"bytes"
	"context"
	"errors"

	"github.com/lestrrat-go/helium/internal/encoding"
	"github.com/lestrrat-go/helium/internal/strcursor"
	"github.com/lestrrat-go/helium/sax"
)

func (pctx *parserCtx) parseDocument(ctx context.Context) error {
	// Store pctx in the context so SAX callbacks (e.g. TreeBuilder) and
	// helium.StopParser can retrieve it via getParserCtx. Also store the
	// document locator.
	ctx = withParserCtx(ctx, pctx)
	ctx = sax.WithDocumentLocator(ctx, pctx)

	// Honor a context that is already cancelled before any parsing work
	// (or blocking reads) begins.
	if err := ctx.Err(); err != nil {
		return err
	}

	if s := pctx.sax; s != nil {
		switch err := s.SetDocumentLocator(ctx, pctx); err {
		case nil, sax.ErrHandlerUnspecified:
			// no op
		default:
			return pctx.error(ctx, err)
		}
	}

	// see if we can find the preliminary encoding
	if pctx.encoding == "" {
		if enc, err := pctx.detectEncoding(); err == nil {
			pctx.detectedEncoding = enc
		}
	}

	// At this stage we MUST be using a ByteCursor, as we
	// don't know what the encoding is.
	bcur := pctx.getByteCursor()
	if bcur == nil {
		return pctx.error(ctx, ErrByteCursorRequired)
	}

	// nothing left? eek
	if bcur.Done() {
		// An apparently empty document may instead be a cancelled read: the
		// push parser's stream returns the context error from its blocking
		// wait, which the byte cursor records and Done() then masks. Surface
		// cancellation (and any other sticky read error) instead of the
		// misleading "empty document".
		if err := ctx.Err(); err != nil {
			return err
		}
		if err := bcur.Err(); err != nil {
			return pctx.error(ctx, err)
		}
		return pctx.error(ctx, errors.New("empty document"))
	}

	// For UTF-16 detected encodings, we must switch encoding FIRST
	// because the XML declaration is encoded in UTF-16, not ASCII.
	switch pctx.detectedEncoding {
	case encUTF16LE, encUTF16BE, encUCS4BE, encUCS4LE, encUCS42143, encUCS43412:
		// For UTF-16 and UCS-4 detected encodings, we must switch encoding
		// FIRST because the XML declaration is encoded in that fixed-width
		// encoding, not ASCII. These encodings are not ASCII-compatible, so
		// the byte-level XML declaration parser cannot read them. The UCS-4
		// detection patterns are the encoded first '<' character (peeked, not
		// consumed), so the decoded cursor still begins at the document start.
		if err := pctx.switchEncoding(); err != nil {
			return pctx.error(ctx, err)
		}
		cur := pctx.getCursor()
		if cur != nil && looksLikeXMLDeclString(cur) {
			if err := pctx.parseXMLDeclFromCursor(ctx); err != nil {
				return pctx.error(ctx, err)
			}
		}
	case encEBCDIC:
		// EBCDIC bytes are not ASCII-compatible, so we cannot parse the
		// XML declaration at byte level. Instead, scan the raw bytes
		// using the EBCDIC invariant character set (shared across all
		// EBCDIC Latin code pages) to extract the encoding name.
		if pctx.rawInput != nil {
			if encName := encoding.ExtractEBCDICEncoding(pctx.rawInput); encName != "" {
				pctx.encoding = encName
			}
		}
		// Fall back to IBM-037 (US EBCDIC) if no encoding was declared.
		if pctx.encoding == "" {
			pctx.encoding = "ibm037"
		}
		// For the byte-slice path rawInput holds the whole document, so reset the
		// byte cursor from it to read the decoder from the document start.
		// For the streaming path (ebcdicStream) rawInput is only a bounded sniff
		// prefix; the live cursor already sits at the document start (detection
		// only peeks) over the full prefix+remainder stream, so decode it in
		// place — the parser's per-node content caps then bound memory as the
		// stream is consumed, with no whole-document buffer.
		if !pctx.ebcdicStream {
			pctx.popInput()
			pctx.pushInput(strcursor.NewByteCursor(bytes.NewReader(pctx.rawInput)))
		}
		if err := pctx.switchEncoding(); err != nil {
			return pctx.error(ctx, err)
		}
		// Parse the XML declaration from the decoded rune cursor.
		cur := pctx.getCursor()
		if cur != nil && looksLikeXMLDeclString(cur) {
			if err := pctx.parseXMLDeclFromCursor(ctx); err != nil {
				return pctx.error(ctx, err)
			}
		}
	default:
		// XML prolog (byte-level for ASCII-compatible encodings)
		if looksLikeXMLDecl(bcur) {
			if err := pctx.parseXMLDecl(ctx); err != nil {
				return pctx.error(ctx, err)
			}
		}

		// At this point we know the encoding, so switch the encoding
		// of the source.
		if err := pctx.switchEncoding(); err != nil {
			return pctx.error(ctx, err)
		}
	}

	// A leading byte-order mark asserts the entity's encoding; a contradicting
	// encoding declaration is a fatal error (XML §4.3.3).
	if err := pctx.checkBOMEncodingConflict(); err != nil {
		return pctx.error(ctx, err)
	}

	if pctx.treeBuilder != nil {
		pctx.fastStartDocument()
	} else if s := pctx.sax; s != nil {
		switch err := s.StartDocument(ctx); err {
		case nil, sax.ErrHandlerUnspecified:
			// no op
		default:
			return pctx.error(ctx, err)
		}
	}

	if pctx.stopped {
		return errParserStopped
	}

	// Misc part of the prolog
	if err := pctx.parseMisc(ctx); err != nil {
		return pctx.error(ctx, err)
	}

	cur := pctx.getCursor()
	if cur == nil {
		return pctx.error(ctx, errNoCursor)
	}
	// Doctype declarations and more misc
	if cur.HasPrefixString("<!DOCTYPE") {
		pctx.inSubset = inInternalSubset
		if err := pctx.parseDocTypeDecl(ctx); err != nil {
			return pctx.error(ctx, err)
		}

		if cur.HasPrefixString("[") {
			pctx.instate = psDTD
			if err := pctx.parseInternalSubset(ctx); err != nil {
				return pctx.error(ctx, err)
			}
		}

		// Query SAX callbacks for subset/standalone status.
		// These mirror libxml2's calls after internal subset parsing.
		if s := pctx.sax; s != nil {
			if has, err := s.HasInternalSubset(ctx); err == nil {
				_ = has // informational; handler may use for validation decisions
			}
			if has, err := s.HasExternalSubset(ctx); err == nil {
				_ = has
			}
		}

		pctx.inSubset = inExternalSubset
		if s := pctx.sax; s != nil {
			switch err := s.ExternalSubset(ctx, pctx.intSubName, pctx.extSubSystem, pctx.extSubURI); err {
			case nil, sax.ErrHandlerUnspecified:
				// no op
			default:
				return pctx.error(ctx, err)
			}
		}
		if pctx.instate == psEOF {
			return pctx.error(ctx, errors.New("unexpected EOF"))
		}
		pctx.inSubset = notInSubset

		// The whole DTD is parsed now, so the entity tables are complete. Re-check
		// every attribute default value's WFCs to catch a nested external/unparsed
		// (or '<') entity reached through a FORWARD-referenced entity that was not
		// yet declared when the default was parsed (SubstituteEntities(false)).
		if err := pctx.validateAttributeDefaultsWFC(ctx); err != nil {
			return err
		}

		pctx.cleanSpecialAttributes()

		pctx.instate = psPrologue
		if err := pctx.parseMisc(ctx); err != nil {
			return pctx.error(ctx, err)
		}
	}
	pctx.skipBlanks(ctx)
	if pctx.blankRunErr != nil {
		return pctx.error(ctx, pctx.blankRunErr)
	}

	if cur.Peek() != '<' {
		// A cancelled or failed read can leave the cursor at an apparent end
		// of input; report the underlying cause, in place of a misleading
		// "start tag expected".
		if err := ctx.Err(); err != nil {
			return err
		}
		if err := pctx.cursorDecodeErr(); err != nil {
			return pctx.error(ctx, err)
		}
		return pctx.error(ctx, ErrEmptyDocument)
	}

	u8, err := pctx.contentCursor()
	if err != nil {
		return pctx.error(ctx, err)
	}
	pctx.instate = psContent
	pctx.armBodyPoll(ctx)
	err = pctx.parseElement(ctx, u8)
	pctx.bodyNeverDone = false
	if err != nil {
		return pctx.error(ctx, err)
	}

	if pctx.stopped {
		return errParserStopped
	}

	pctx.instate = psEpilogue

	if err := pctx.parseMisc(ctx); err != nil {
		return pctx.error(ctx, err)
	}
	if !cur.Done() {
		return pctx.error(ctx, ErrDocumentEnd)
	}
	// A clean Done() may mask a transcoding/decode error (e.g. an unpaired
	// UTF-16 surrogate the decoder replaced with U+FFFD). Surface it as a fatal
	// error, so truncated or malformed encoded input is never accepted.
	if err := pctx.cursorDecodeErr(); err != nil {
		return pctx.error(ctx, err)
	}
	pctx.instate = psEOF

	// All done
	if pctx.treeBuilder != nil {
		pctx.fastEndDocument()
	} else if s := pctx.sax; s != nil {
		switch err := s.EndDocument(ctx); err {
		case nil, sax.ErrHandlerUnspecified:
			// no op
		default:
			return pctx.error(ctx, err)
		}
	}

	return nil
}

// contentCursor returns the input cursor that element content is parsed from.
// Every parse installs a UTF-8 cursor (switchEncoding) before it reaches
// element content, so the element-content functions take the concrete cursor
// as a parameter: its methods are called directly, and inlined where small,
// instead of through strcursor.Cursor, and no level re-fetches it from the
// input stack.
func (pctx *parserCtx) contentCursor() (*strcursor.UTF8Cursor, error) {
	cur := pctx.getCursor()
	if cur == nil {
		return nil, errNoCursor
	}
	u8, ok := cur.(*strcursor.UTF8Cursor)
	if !ok {
		return nil, errContentCursor
	}
	return u8, nil
}

// parseContentInput parses element content from the current input: the entry
// point of a sub-parse (entity replacement text, a node-context fragment).
func (pctx *parserCtx) parseContentInput(ctx context.Context) error {
	cur, err := pctx.contentCursor()
	if err != nil {
		return pctx.error(ctx, err)
	}
	return pctx.parseContent(ctx, cur)
}

// parseContent parses element content from cur, the cursor contentCursor
// returned. A reference is expanded in its own parser context, so the input
// stack, and with it cur, does not change while the loop runs.
func (pctx *parserCtx) parseContent(ctx context.Context, cur *strcursor.UTF8Cursor) error {
	pctx.instate = psContent

	doRecover := pctx.options.IsSet(parseRecover)

	for {
		// Check the context BEFORE cur.Done(), which may refill the cursor
		// from an io.Reader and block; this lets a cancelled context be
		// observed between reads, ahead of any blocking refill.
		if err := pctx.pollErr(ctx); err != nil {
			return err
		}
		if cur.Done() || pctx.stopped {
			break
		}
		if cur.Peek() == '<' && cur.PeekAt(1) == '/' {
			break
		}

		var err error
		switch cur.Peek() {
		case '<':
			switch cur.PeekAt(1) {
			case '?':
				err = pctx.parsePI(ctx)
			case '!':
				switch {
				case cur.PeekAt(2) == '[' &&
					cur.PeekAt(3) == 'C' &&
					cur.PeekAt(4) == 'D' &&
					cur.PeekAt(5) == 'A' &&
					cur.PeekAt(6) == 'T' &&
					cur.PeekAt(7) == 'A' &&
					cur.PeekAt(8) == '[':
					err = pctx.parseCDSect(ctx)
				case cur.PeekAt(2) == '-' && cur.PeekAt(3) == '-':
					err = pctx.parseComment(ctx)
				default:
					err = pctx.parseElement(ctx, cur)
				}
			default:
				err = pctx.parseElement(ctx, cur)
			}
		case '&':
			err = pctx.parseReference(ctx)
		default:
			if err := pctx.parseCharDataContent(ctx, cur); err != nil {
				if !doRecover || isParseAbort(err) {
					return err
				}
				if pctx.recoverErr == nil {
					pctx.recoverErr = err
				}
				pctx.disableSAX = true
				pctx.wellFormed = false
				if err := pctx.skipToRecoverPoint(ctx); err != nil {
					return err
				}
			}
			continue
		}

		if err != nil {
			if !doRecover || isParseAbort(err) {
				return pctx.error(ctx, err)
			}
			if pctx.recoverErr == nil {
				pctx.recoverErr = err
			}
			pctx.disableSAX = true
			pctx.wellFormed = false

			prevLine, prevCol := cur.LineNumber(), cur.Column()
			if err := pctx.skipToRecoverPoint(ctx); err != nil {
				return err
			}
			if !cur.Done() && cur.LineNumber() == prevLine && cur.Column() == prevCol {
				_ = cur.Advance(1)
			}
			continue
		}
	}

	if pctx.stopped {
		return errParserStopped
	}

	if pctx.recoverErr != nil {
		return pctx.recoverErr
	}

	return nil
}

// skipToRecoverPoint advances the cursor past unrecoverable content to the
// next '<' character or EOF, for re-synchronization in parseRecover mode.
// It returns the context error if the context is cancelled so that recovery
// cannot run (or block on a cursor refill) after cancellation.
func (pctx *parserCtx) skipToRecoverPoint(ctx context.Context) error {
	cur := pctx.getCursor()
	if cur == nil {
		return nil
	}
	for {
		// Check the context BEFORE cur.Done(), which may refill the cursor
		// from an io.Reader and block; this lets a cancelled context abort
		// recovery, with no spin and no block.
		if err := ctx.Err(); err != nil {
			return err
		}
		if cur.Done() {
			return nil
		}
		if cur.Peek() == '<' {
			return nil
		}
		_ = cur.Advance(1)
	}
}