// ===========================================================================
//
//                            PUBLIC DOMAIN NOTICE
//            National Center for Biotechnology Information (NCBI)
//
//  This software/database is a "United States Government Work" under the
//  terms of the United States Copyright Act. It was written as part of
//  the author's official duties as a United States Government employee and
//  thus cannot be copyrighted. This software/database is freely available
//  to the public for use. The National Library of Medicine and the U.S.
//  Government do not place any restriction on its use or reproduction.
//  We would, however, appreciate having the NCBI and the author cited in
//  any work or product based on this material.
//
//  Although all reasonable efforts have been taken to ensure the accuracy
//  and reliability of the software and data, the NLM and the U.S.
//  Government do not and cannot warrant the performance or results that
//  may be obtained by using this software or data. The NLM and the U.S.
//  Government disclaim all warranties, express or implied, including
//  warranties of performance, merchantability or fitness for any particular
//  purpose.
//
// ===========================================================================
//
// File Name:  transmute.go
//
// Author:  Jonathan Kans
//
// ==========================================================================

package main

import (
	"bufio"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/gedex/inflector"
	"github.com/klauspost/cpuid"
	"github.com/pbnjay/memory"
	"html"
	"io"
	"net/url"
	"os"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
)

// TRANSMUTE VERSION AND HELP MESSAGE TEXT

const transmuteVersion = "14.0"

const transmuteHelp = `
Pretty-Printing

  -j2p          Reformat JSON

  -x2p          Reformat XML

Customized XML Reformatting

  -format       [copy|compact|flush|indent|expand]
                  [-xml]
                  [-doctype]
                  [-comment]
                  [-cdata]
                  [-unicode fuse|space|period|brackets|markdown|slash|tag]
                  [-script brackets|markdown]
                  [-mathml terse]

XML Modification

  -filter       Object
                   [retain|remove|encode|decode|shrink|expand|accent]
                    [content|cdata|comment|object|attributes|container]

EFetch XML Normalization

  -normalize    [database]

Data Conversion

  -j2x          JSON stream to XML, suitable for xtract -path navigation
                  [-set setWrapper]
                  [-rec recordWrapper]
                  [-nest flat|recurse|plural|depth]

  -a2x          ASN.1 stream to XML, suitable for xtract -path navigation
                  [-set setWrapper]
                  [-rec recordWrapper]

  -t2x          Tab-delimited table to XML
                  [-set setWrapper]
                  [-rec recordWrapper]
                  [-skip linesToSkip]
                  [-lower | -upper]
                  [-indent | -flush]
                  XML object names per column

  -c2x          CSV (comma-separated values) file to XML
                  [-set setWrapper]
                  [-rec recordWrapper]
                  [-skip linesToSkip]
                  [-lower | -upper]
                  [-indent | -flush]
                  XML object names per column

  -g2x          GenBank/GenPept flatfile to INSDSeq XML

String Transformations

 XML

  -encodeXML
  -decodeXML

  -plainXML

 URL

  -encodeURL
  -decodeURL

 Base64

  -encodeB64
  -decodeB64

Examples

  -j2x -set - -rec GeneRec

  -t2x -set Set -rec Rec -skip 1 Code Name

  -filter ExpXml decode content

  -filter LocationHist remove object

  -normalize pubmed

  str="the rain & in [spain] stay's <mainly> on \"the\" plain"
  echo "$str"
  echo "$str" | transmute -plainXML
  echo "$str" | transmute -encodeXML
  echo "$str" | transmute -encodeURL
  echo "$str" | transmute -encodeB64
  echo "$str" | transmute -encodeXML | transmute -decodeXML
  echo "$str" | transmute -encodeURL | transmute -decodeURL
  echo "$str" | transmute -encodeB64 | transmute -decodeB64
`

// UTILITIES

func ParseMarkup(str, cmd string) MarkupPolicy {

	switch str {
	case "fuse", "fused":
		return FUSE
	case "space", "spaces":
		return SPACE
	case "period", "periods":
		return PERIOD
	case "bracket", "brackets":
		return BRACKETS
	case "markdown":
		return MARKDOWN
	case "slash":
		return SLASH
	case "tag", "tags":
		return TAGS
	case "terse":
		return TERSE
	default:
		if str != "" {
			fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized %s value '%s'\n", cmd, str)
			os.Exit(1)
		}
	}
	return NOMARKUP
}

// XML FORMATTING FUNCTIONS

func CreateFormatters(parent string, inp <-chan Extract) <-chan Extract {

	if inp == nil {
		return nil
	}

	out := make(chan Extract, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create formatter channel\n")
		os.Exit(1)
	}

	// xmlFormatter reads partitioned XML from channel and formats on a per-record basis
	xmlFormatter := func(wg *sync.WaitGroup, inp <-chan Extract, out chan<- Extract) {

		// report when this formatter has no more records to process
		defer wg.Done()

		// clean and reformat one record, flush left
		doFormat := func(text string) string {

			var buffer strings.Builder

			// print attributes
			printAttributes := func(attr string) {

				if attr == "" {
					return
				}
				attr = strings.TrimSpace(attr)
				attr = CompressRunsOfSpaces(attr)

				if DeAccent {
					if IsNotASCII(attr) {
						attr = DoAccentTransform(attr)
					}
				}
				if DoASCII {
					if IsNotASCII(attr) {
						attr = UnicodeToASCII(attr)
					}
				}

				buffer.WriteString(" ")
				buffer.WriteString(attr)
			}

			ret := "\n"
			pfx := ""
			skip := 0

			doCleanup := func(tkn Token, nxtTag TagType, nxtName string) {

				if skip > 0 {
					skip--
					return
				}

				name := tkn.Name

				switch tkn.Tag {
				case STARTTAG:
					// convert start-stop to self-closing tag if attributes are present, otherwise skip
					if nxtTag == STOPTAG && nxtName == name {
						if tkn.Attr != "" {
							buffer.WriteString(pfx)
							buffer.WriteString("<")
							buffer.WriteString(name)
							printAttributes(tkn.Attr)
							buffer.WriteString("/>")
							pfx = ret
						}
						skip++
						return
					}
					buffer.WriteString(pfx)
					buffer.WriteString("<")
					buffer.WriteString(name)
					printAttributes(tkn.Attr)
					buffer.WriteString(">")
					pfx = ret
				case SELFTAG:
					if tkn.Attr != "" {
						buffer.WriteString(pfx)
						buffer.WriteString("<")
						buffer.WriteString(name)
						printAttributes(tkn.Attr)
						buffer.WriteString("/>")
						pfx = ret
					}
				case STOPTAG:
					buffer.WriteString(pfx)
					buffer.WriteString("</")
					buffer.WriteString(name)
					buffer.WriteString(">")
					if DoMixed && nxtTag == CONTENTTAG {
						buffer.WriteString(" ")
					}
					pfx = ret
				case CONTENTTAG:
					if nxtTag == STARTTAG || nxtTag == SELFTAG {
						if DoStrict {
							fmt.Fprintf(os.Stderr, "ERROR: UNRECOGNIZED MIXED CONTENT <%s> in <%s>\n", nxtName, name)
						} else if !DoMixed {
							fmt.Fprintf(os.Stderr, "ERROR: UNEXPECTED MIXED CONTENT <%s> in <%s>\n", nxtName, name)
						}
					}
					if len(name) > 0 && IsNotJustWhitespace(name) {
						// support for all content processing flags
						if DoStrict || DoMixed || DoCompress {
							ctype := tkn.Cont
							name = CleanupContents(name, (ctype&ASCII) != 0, (ctype&AMPER) != 0, (ctype&MIXED) != 0)
						}
						buffer.WriteString(name)
					}
					if DoMixed && nxtTag == STARTTAG {
						buffer.WriteString(" ")
					}
					pfx = ""
				case CDATATAG, COMMENTTAG:
					// ignore
				case DOCTYPETAG:
				case NOTAG:
				case ISCLOSED:
					// now handled at end of calling function
				default:
					buffer.WriteString(pfx)
					pfx = ""
				}
			}

			var prev Token
			primed := false

			// track adjacent pairs to give look-ahead at next token
			doPair := func(tkn Token) {

				if primed {
					doCleanup(prev, tkn.Tag, tkn.Name)
				}

				prev = Token{tkn.Tag, tkn.Cont, tkn.Name, tkn.Attr, tkn.Index, tkn.Line}
				primed = true
			}

			ParseXML(text, parent, nil, doPair, nil, nil)

			// isclosed tag
			if primed {
				buffer.WriteString(pfx)
			}

			txt := buffer.String()

			if DoMixed {
				// clean up reconstructed mixed content
				txt = DoTrimFlankingHTML(txt)
				if HasBadSpace(txt) {
					txt = CleanupBadSpaces(txt)
				}
				if HasAdjacentSpaces(txt) {
					txt = CompressRunsOfSpaces(txt)
				}
				if NeedsTightening(txt) {
					txt = TightenParentheses(txt)
				}
			}

			return txt
		}

		// read partitioned XML from producer channel
		for ext := range inp {

			idx := ext.Index
			text := ext.Text

			if text == "" {
				// should never see empty input data
				out <- Extract{idx, "", text, nil}
				continue
			}

			str := doFormat(text[:])

			// send even if empty to get all record counts for reordering
			out <- Extract{idx, "", str, nil}
		}
	}

	var wg sync.WaitGroup

	// launch multiple formatter goroutines
	for i := 0; i < NumServe; i++ {
		wg.Add(1)
		go xmlFormatter(&wg, inp, out)
	}

	// launch separate anonymous goroutine to wait until all formatters are done
	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

// StreamBlocks -> StreamTokens -> ProcessStream

// ProcessTokens shows individual tokens in stream (undocumented)
func ProcessTokens(rdr <-chan string) {

	if rdr == nil {
		return
	}

	tknq := CreateTokenizer(rdr)

	if tknq == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create debug tokenizer\n")
		os.Exit(1)
	}

	var buffer strings.Builder

	count := 0
	indent := 0

	for tkn := range tknq {

		tag := tkn.Tag
		name := tkn.Name
		attr := tkn.Attr

		switch tag {
		case STARTTAG:
			buffer.WriteString("ST: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
			if attr != "" {
				buffer.WriteString("AT: ")
				for i := 0; i < indent; i++ {
					buffer.WriteString("  ")
				}
				buffer.WriteString(attr)
				buffer.WriteString("\n")
			}
			indent++
		case SELFTAG:
			buffer.WriteString("SL: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("/")
			buffer.WriteString("\n")
			if attr != "" {
				buffer.WriteString("AT: ")
				for i := 0; i < indent; i++ {
					buffer.WriteString("  ")
				}
				buffer.WriteString(attr)
				buffer.WriteString("\n")
			}
		case STOPTAG:
			indent--
			buffer.WriteString("SP: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("/")
			buffer.WriteString("\n")
		case CONTENTTAG:
			ctype := tkn.Cont
			if (ctype & LFTSPACE) != 0 {
				if (ctype & RGTSPACE) != 0 {
					buffer.WriteString("FL: ")
				} else {
					buffer.WriteString("LF: ")
				}
			} else if (ctype & RGTSPACE) != 0 {
				buffer.WriteString("RT: ")
			} else {
				buffer.WriteString("VL: ")
			}
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case CDATATAG:
			buffer.WriteString("CD: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case COMMENTTAG:
			buffer.WriteString("CO: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case DOCTYPETAG:
			buffer.WriteString("DC: ")
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case NOTAG:
			buffer.WriteString("NO:")
			if indent != 0 {
				buffer.WriteString(" (indent ")
				buffer.WriteString(strconv.Itoa(indent))
				buffer.WriteString(")")
			}
			buffer.WriteString("\n")
		case ISCLOSED:
			buffer.WriteString("CL:")
			if indent != 0 {
				buffer.WriteString(" (indent ")
				buffer.WriteString(strconv.Itoa(indent))
				buffer.WriteString(")")
			}
			buffer.WriteString("\n")
			txt := buffer.String()
			if txt != "" {
				// print final buffer
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			return
		default:
			buffer.WriteString("UNKONWN:")
			if indent != 0 {
				buffer.WriteString(" (indent ")
				buffer.WriteString(strconv.Itoa(indent))
				buffer.WriteString(")")
			}
			buffer.WriteString("\n")
		}

		count++
		if count > 1000 {
			count = 0
			txt := buffer.String()
			if txt != "" {
				// print current buffered output
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			buffer.Reset()
		}
	}
}

// ProcessFormat reformats XML for ease of reading
func ProcessFormat(rdr <-chan string, args []string, useTimer bool) int {

	if rdr == nil || args == nil {
		return 0
	}

	var buffer strings.Builder
	count := 0
	maxLine := 0

	// skip past command name
	args = args[1:]

	copyRecrd := false
	compRecrd := false
	flushLeft := false
	wrapAttrs := false
	ret := "\n"

	xml := ""
	customDoctype := false
	doctype := ""

	doComment := false
	doCdata := false

	// look for [copy|compact|flush|indent|expand] specification
	if len(args) > 0 {
		inSwitch := true

		switch args[0] {
		case "compact", "compacted", "compress", "compressed", "terse", "*":
			// compress to one record per line
			compRecrd = true
			ret = ""
		case "flush", "flushed", "left":
			// suppress line indentation
			flushLeft = true
		case "expand", "expanded", "extend", "extended", "verbose", "@":
			// each attribute on its own line
			wrapAttrs = true
		case "indent", "indented", "normal", "default":
			// default behavior
		case "copy":
			// fast block copy
			copyRecrd = true
		default:
			// if not any of the controls, will check later for -xml and -doctype arguments
			inSwitch = false
		}

		if inSwitch {
			// skip past first argument
			args = args[1:]
		}
	}

	// fast block copy, supporting only -spaces processing flag
	if copyRecrd {

		for str := range rdr {
			if str == "" {
				break
			}

			os.Stdout.WriteString(str)
		}
		os.Stdout.WriteString("\n")
		return 0
	}

	unicodePolicy := ""
	scriptPolicy := ""
	mathmlPolicy := ""

	fuseTopSets := true
	keepEmptySelfClosing := false

	// look for -xml and -doctype arguments (undocumented)
	for len(args) > 0 {

		switch args[0] {
		case "-xml":
			args = args[1:]
			// -xml argument must be followed by value to use in xml line
			if len(args) < 1 || strings.HasPrefix(args[0], "-") {
				fmt.Fprintf(os.Stderr, "\nERROR: -xml argument is missing\n")
				os.Exit(1)
			}
			xml = args[0]
			args = args[1:]
		case "-doctype":
			customDoctype = true
			args = args[1:]
			if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
				// if -doctype argument followed by value, use instead of DOCTYPE line
				doctype = args[0]
				args = args[1:]
			}

		// allow setting of unicode, script, and mathml flags within -format
		case "-unicode":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Unicode argument is missing\n")
				os.Exit(1)
			}
			unicodePolicy = args[1]
			args = args[2:]
		case "-script":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Script argument is missing\n")
				os.Exit(1)
			}
			scriptPolicy = args[1]
			args = args[2:]
		case "-mathml":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: MathML argument is missing\n")
				os.Exit(1)
			}
			mathmlPolicy = args[1]
			args = args[2:]

		// also allow setting additional processing flags within -format (undocumented)
		case "-strict":
			// can set -strict within -format
			DoStrict = true
			args = args[1:]
		case "-mixed":
			// can set -mixed within -format
			DoMixed = true
			args = args[1:]
		case "-compress", "-compressed":
			DoCompress = true
			args = args[1:]
		case "-accent":
			DeAccent = true
			args = args[1:]
		case "-ascii":
			DoASCII = true
			args = args[1:]
		case "-separate", "-separated":
			fuseTopSets = false
			args = args[1:]
		case "-self", "-self-closing":
			keepEmptySelfClosing = true
			args = args[1:]
		case "-comment":
			doComment = true
			args = args[1:]
		case "-cdata":
			doCdata = true
			args = args[1:]
		default:
			fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized option after -format command\n")
			os.Exit(1)
		}
	}

	UnicodeFix = ParseMarkup(unicodePolicy, "-unicode")
	ScriptFix = ParseMarkup(scriptPolicy, "-script")
	MathMLFix = ParseMarkup(mathmlPolicy, "-mathml")

	if UnicodeFix != NOMARKUP {
		DoUnicode = true
	}

	if ScriptFix != NOMARKUP {
		DoScript = true
	}

	if MathMLFix != NOMARKUP {
		DoMathML = true
	}

	CountLines = DoMixed || useTimer
	AllowEmbed = DoStrict || DoMixed
	ContentMods = AllowEmbed || DoCompress || DoUnicode || DoScript || DoMathML || DeAccent || DoASCII

	// array to speed up indentation
	indentSpaces := []string{
		"",
		"  ",
		"    ",
		"      ",
		"        ",
		"          ",
		"            ",
		"              ",
		"                ",
		"                  ",
	}

	indent := 0

	// indent a specified number of spaces
	doIndent := func(indt int) {
		if compRecrd || flushLeft {
			return
		}
		i := indt
		for i > 9 {
			buffer.WriteString("                    ")
			i -= 10
		}
		if i < 0 {
			return
		}
		buffer.WriteString(indentSpaces[i])
	}

	badAttributeSpacing := func(attr string) bool {

		if len(attr) < 2 {
			return false
		}

		var prev rune

		for _, ch := range attr {
			if ch == '=' && prev == ' ' {
				return true
			}
			if ch == ' ' && prev == '=' {
				return true
			}
			prev = ch
		}

		return false
	}

	// print attributes
	printAttributes := func(attr string) {

		if attr == "" {
			return
		}
		attr = strings.TrimSpace(attr)
		attr = CompressRunsOfSpaces(attr)

		if DeAccent {
			if IsNotASCII(attr) {
				attr = DoAccentTransform(attr)
			}
		}
		if DoASCII {
			if IsNotASCII(attr) {
				attr = UnicodeToASCII(attr)
			}
		}

		if wrapAttrs {

			start := 0
			idx := 0
			inQuote := false

			attlen := len(attr)

			for idx < attlen {
				ch := attr[idx]
				if ch == '=' && !inQuote {
					inQuote = true
					str := strings.TrimSpace(attr[start:idx])
					buffer.WriteString("\n")
					doIndent(indent)
					buffer.WriteString(" ")
					buffer.WriteString(str)
					// skip past equal sign
					idx++
					ch = attr[idx]
					if ch != '"' && ch != '\'' {
						// "
						// skip past unexpected blanks
						for InBlank[ch] {
							idx++
							ch = attr[idx]
						}
					}
					// skip past leading double quote
					idx++
					start = idx
				} else if ch == '"' || ch == '\'' {
					// "
					inQuote = !inQuote
					str := strings.TrimSpace(attr[start:idx])
					buffer.WriteString("=\"")
					buffer.WriteString(str)
					buffer.WriteString("\"")
					// skip past trailing double quote and (possible) space
					idx += 2
					start = idx
				} else {
					idx++
				}
			}

			buffer.WriteString("\n")
			doIndent(indent)

		} else if badAttributeSpacing(attr) {

			buffer.WriteString(" ")

			start := 0
			idx := 0

			attlen := len(attr)

			for idx < attlen {
				ch := attr[idx]
				if ch == '=' {
					str := strings.TrimSpace(attr[start:idx])
					buffer.WriteString(str)
					// skip past equal sign
					idx++
					ch = attr[idx]
					if ch != '"' && ch != '\'' {
						// "
						// skip past unexpected blanks
						for InBlank[ch] {
							idx++
							ch = attr[idx]
						}
					}
					// skip past leading double quote
					idx++
					start = idx
				} else if ch == '"' || ch == '\'' {
					// "
					str := strings.TrimSpace(attr[start:idx])
					buffer.WriteString("=\"")
					buffer.WriteString(str)
					buffer.WriteString("\"")
					// skip past trailing double quote and (possible) space
					idx += 2
					start = idx
				} else {
					idx++
				}
			}

		} else {

			buffer.WriteString(" ")
			buffer.WriteString(attr)
		}
	}

	parent := ""
	pfx := ""
	skip := 0
	okIndent := true

	printXMLAndDoctype := func(xml, doctype, parent string) {

		// check for xml line explicitly set in argument
		if xml != "" {
			xml = strings.TrimSpace(xml)
			if strings.HasPrefix(xml, "<") {
				xml = xml[1:]
			}
			if strings.HasPrefix(xml, "?") {
				xml = xml[1:]
			}
			if strings.HasPrefix(xml, "xml") {
				xml = xml[3:]
			}
			if strings.HasPrefix(xml, " ") {
				xml = xml[1:]
			}
			if strings.HasSuffix(xml, "?>") {
				xlen := len(xml)
				xml = xml[:xlen-2]
			}
			xml = strings.TrimSpace(xml)

			buffer.WriteString("<?xml ")
			buffer.WriteString(xml)
			buffer.WriteString("?>")
		} else {
			buffer.WriteString("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
		}

		buffer.WriteString("\n")

		// check for doctype taken from XML file or explicitly set in argument
		if doctype != "" {
			doctype = strings.TrimSpace(doctype)
			if strings.HasPrefix(doctype, "<") {
				doctype = doctype[1:]
			}
			if strings.HasPrefix(doctype, "!") {
				doctype = doctype[1:]
			}
			if strings.HasPrefix(doctype, "DOCTYPE") {
				doctype = doctype[7:]
			}
			if strings.HasPrefix(doctype, " ") {
				doctype = doctype[1:]
			}
			doctype = strings.TrimSuffix(doctype, ">")
			doctype = strings.TrimSpace(doctype)

			buffer.WriteString("<!DOCTYPE ")
			buffer.WriteString(doctype)
			buffer.WriteString(">")
		} else {
			buffer.WriteString("<!DOCTYPE ")
			buffer.WriteString(parent)
			buffer.WriteString(">")
		}

		buffer.WriteString("\n")
	}

	cleanupMixed := func(txt string) string {

		txt = DoTrimFlankingHTML(txt)
		if HasBadSpace(txt) {
			txt = CleanupBadSpaces(txt)
		}
		if HasAdjacentSpaces(txt) {
			txt = CompressRunsOfSpaces(txt)
		}
		if NeedsTightening(txt) {
			txt = TightenParentheses(txt)
		}

		return txt
	}

	doCleanup := func(tkn Token, nxtTag TagType, nxtName, nxtAttr string) {

		if skip > 0 {
			skip--
			return
		}

		name := tkn.Name

		switch tkn.Tag {
		case STARTTAG:
			// detect first start tag, print xml and doctype parent
			if indent == 0 && parent == "" {
				parent = name
				printXMLAndDoctype(xml, doctype, parent)
				// do not fuse <opt> or <anon> top-level objects (converted from JSON)
				if parent == "opt" || parent == "anon" {
					fuseTopSets = false
				}
			}
			// convert start-stop to self-closing tag if attributes are present, otherwise skip
			if nxtTag == STOPTAG && nxtName == name {
				if tkn.Attr != "" || keepEmptySelfClosing {
					buffer.WriteString(pfx)
					doIndent(indent)
					buffer.WriteString("<")
					buffer.WriteString(name)
					printAttributes(tkn.Attr)
					buffer.WriteString("/>")
					pfx = ret
					okIndent = true
				}
				skip++
				return
			}
			buffer.WriteString(pfx)
			doIndent(indent)
			indent++
			buffer.WriteString("<")
			buffer.WriteString(name)
			printAttributes(tkn.Attr)
			buffer.WriteString(">")
			pfx = ret
			okIndent = true
			if compRecrd && indent == 1 {
				buffer.WriteString("\n")
			}
		case SELFTAG:
			if tkn.Attr != "" || keepEmptySelfClosing {
				buffer.WriteString(pfx)
				doIndent(indent)
				buffer.WriteString("<")
				buffer.WriteString(name)
				printAttributes(tkn.Attr)
				buffer.WriteString("/>")
				pfx = ret
				okIndent = true
			}
		case STOPTAG:
			// skip internal copies of top-level </parent><parent> tags (to fuse multiple chunks returned by efetch)
			// do not skip if attributes are present, unless pattern ends in "Set" (e.g., <DocumentSummarySet status="OK">)
			if nxtTag == STARTTAG && indent == 1 && fuseTopSets && nxtName == parent {
				if nxtAttr == "" || (strings.HasSuffix(parent, "Set") && len(parent) > 3) {
					skip++
					return
				}
			}
			buffer.WriteString(pfx)
			indent--
			if okIndent {
				doIndent(indent)
			}
			buffer.WriteString("</")
			buffer.WriteString(name)
			buffer.WriteString(">")
			if DoMixed && nxtTag == CONTENTTAG {
				buffer.WriteString(" ")
			}
			pfx = ret
			okIndent = true
			if compRecrd && indent < 2 {
				buffer.WriteString("\n")
			}
		case CONTENTTAG:
			if nxtTag == STARTTAG || nxtTag == SELFTAG {
				if DoStrict {
					fmt.Fprintf(os.Stderr, "ERROR: UNRECOGNIZED MIXED CONTENT <%s> in <%s>\n", nxtName, name)
				} else if !DoMixed {
					fmt.Fprintf(os.Stderr, "ERROR: UNEXPECTED MIXED CONTENT <%s> in <%s>\n", nxtName, name)
				}
			}
			if len(name) > 0 && IsNotJustWhitespace(name) {
				// support for all content processing flags
				if DoStrict || DoMixed || DoCompress {
					ctype := tkn.Cont
					name = CleanupContents(name, (ctype&ASCII) != 0, (ctype&AMPER) != 0, (ctype&MIXED) != 0)
				}
				buffer.WriteString(name)
			}
			if DoMixed && nxtTag == STARTTAG {
				buffer.WriteString(" ")
			}
			pfx = ""
			okIndent = false
		case CDATATAG:
			if doCdata {
				buffer.WriteString(pfx)
				doIndent(indent)
				buffer.WriteString("<![CDATA[")
				buffer.WriteString(name)
				buffer.WriteString("]]>")
				pfx = ret
				okIndent = true
			}
		case COMMENTTAG:
			if doComment {
				buffer.WriteString(pfx)
				doIndent(indent)
				buffer.WriteString("<!--")
				buffer.WriteString(name)
				buffer.WriteString("-->")
				pfx = ret
				okIndent = true
			}
		case DOCTYPETAG:
			if customDoctype && doctype == "" {
				doctype = name
			}
		case NOTAG:
		case ISCLOSED:
			// now handled at end of calling function
		default:
			buffer.WriteString(pfx)
			pfx = ""
			okIndent = false
		}

		count++
		if count > 1000 {
			count = 0
			txt := buffer.String()
			if DoMixed {
				txt = cleanupMixed(txt)
			}
			if txt != "" {
				// print current buffered output
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			buffer.Reset()
		}
	}

	var prev Token
	primed := false
	skipDoctype := false

	// track adjacent pairs to give look-ahead at next token
	doPair := func(tkn Token) {

		if tkn.Tag == NOTAG {
			return
		}

		maxLine = tkn.Line

		if tkn.Tag == DOCTYPETAG {
			if skipDoctype {
				return
			}
			skipDoctype = true
		}

		if primed {
			doCleanup(prev, tkn.Tag, tkn.Name, tkn.Attr)
		}

		prev = Token{tkn.Tag, tkn.Cont, tkn.Name, tkn.Attr, tkn.Index, tkn.Line}
		primed = true
	}

	ParseXML("", "", rdr, doPair, nil, nil)

	// handle isclosed tag
	if primed {
		buffer.WriteString(pfx)
		txt := buffer.String()
		if DoMixed {
			txt = cleanupMixed(txt)
		}
		if txt != "" {
			// print final buffer
			fmt.Fprintf(os.Stdout, "%s", txt)
		}
	}

	return maxLine
}

// ProcessOutline displays outline of XML structure
func ProcessOutline(rdr <-chan string) {

	if rdr == nil {
		return
	}

	tknq := CreateTokenizer(rdr)

	if tknq == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create outline tokenizer\n")
		os.Exit(1)
	}

	var buffer strings.Builder

	count := 0
	indent := 0

	for tkn := range tknq {

		tag := tkn.Tag
		name := tkn.Name

		switch tag {
		case STARTTAG:
			if name == "eSummaryResult" ||
				name == "eLinkResult" ||
				name == "eInfoResult" ||
				name == "PubmedArticleSet" ||
				name == "DocumentSummarySet" ||
				name == "INSDSet" ||
				name == "Entrezgene-Set" ||
				name == "TaxaSet" {
				break
			}
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
			indent++
		case SELFTAG:
			for i := 0; i < indent; i++ {
				buffer.WriteString("  ")
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case STOPTAG:
			indent--
		case DOCTYPETAG:
		case NOTAG:
		case ISCLOSED:
			txt := buffer.String()
			if txt != "" {
				// print final buffer
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			return
		default:
		}

		count++
		if count > 1000 {
			count = 0
			txt := buffer.String()
			if txt != "" {
				// print current buffered output
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			buffer.Reset()
		}
	}
}

// ProcessSynopsis displays paths to XML elements
func ProcessSynopsis(rdr <-chan string, leaf bool, delim string) {

	if rdr == nil {
		return
	}

	tknq := CreateTokenizer(rdr)

	if tknq == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create synopsis tokenizer\n")
		os.Exit(1)
	}

	var buffer strings.Builder
	count := 0

	// synopsisLevel recursive definition
	var synopsisLevel func(string) bool

	synopsisLevel = func(parent string) bool {

		for tkn := range tknq {

			tag := tkn.Tag
			name := tkn.Name

			switch tag {
			case STARTTAG:
				if name == "eSummaryResult" ||
					name == "eLinkResult" ||
					name == "eInfoResult" ||
					name == "PubmedArticleSet" ||
					name == "DocumentSummarySet" ||
					name == "INSDSet" ||
					name == "Entrezgene-Set" ||
					name == "TaxaSet" {
					break
				}
				if leaf {
					if name == "root" ||
						name == "opt" ||
						name == "anon" {
						break
					}
				}
				if !leaf {
					// show all paths, including container objects
					if parent != "" {
						buffer.WriteString(parent)
						buffer.WriteString(delim)
					}
					buffer.WriteString(name)
					buffer.WriteString("\n")
				}
				path := parent
				if path != "" {
					path += delim
				}
				path += name
				if synopsisLevel(path) {
					return true
				}
			case SELFTAG:
				if parent != "" {
					buffer.WriteString(parent)
					buffer.WriteString(delim)
				}
				buffer.WriteString(name)
				buffer.WriteString("\n")
			case STOPTAG:
				// break recursion
				return false
			case CONTENTTAG:
				if leaf {
					// only show endpoint paths
					if parent != "" {
						buffer.WriteString(parent)
						buffer.WriteString("\n")
					}
				}
			case DOCTYPETAG:
			case NOTAG:
			case ISCLOSED:
				txt := buffer.String()
				if txt != "" {
					// print final buffer
					fmt.Fprintf(os.Stdout, "%s", txt)
				}
				return true
			default:
			}

			count++
			if count > 1000 {
				count = 0
				txt := buffer.String()
				if txt != "" {
					// print current buffered output
					fmt.Fprintf(os.Stdout, "%s", txt)
				}
				buffer.Reset()
			}
		}
		return true
	}

	for {
		// may have concatenated XMLs, loop through all
		if synopsisLevel("") {
			return
		}
	}
}

// ProcessFilter modifies XML content, comments, or CDATA
func ProcessFilter(rdr <-chan string, args []string) {

	if rdr == nil || args == nil {
		return
	}

	tknq := CreateTokenizer(rdr)

	if tknq == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create filter tokenizer\n")
		os.Exit(1)
	}

	var buffer strings.Builder

	count := 0

	// skip past command name
	args = args[1:]

	max := len(args)
	if max < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to transmute -filter\n")
		os.Exit(1)
	}

	pttrn := args[0]

	args = args[1:]
	max--

	if max < 2 {
		fmt.Fprintf(os.Stderr, "\nERROR: No object name supplied to transmute -filter\n")
		os.Exit(1)
	}

	type ActionType int

	const (
		NOACTION ActionType = iota
		DORETAIN
		DOREMOVE
		DOENCODE
		DODECODE
		DOSHRINK
		DOEXPAND
		DOACCENT
	)

	action := args[0]

	what := NOACTION
	switch action {
	case "retain":
		what = DORETAIN
	case "remove":
		what = DOREMOVE
	case "encode":
		what = DOENCODE
	case "decode":
		what = DODECODE
	case "shrink":
		what = DOSHRINK
	case "expand":
		what = DOEXPAND
	case "accent":
		what = DOACCENT
	default:
		fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized action '%s' supplied to transmute -filter\n", action)
		os.Exit(1)
	}

	trget := args[1]

	which := NOTAG
	switch trget {
	case "attribute", "attributes":
		which = ATTRIBTAG
	case "content", "contents":
		which = CONTENTTAG
	case "cdata", "CDATA":
		which = CDATATAG
	case "comment", "comments":
		which = COMMENTTAG
	case "object":
		// object normally retained
		which = OBJECTTAG
	case "container":
		which = CONTAINERTAG
	default:
		fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized target '%s' supplied to transmute -filter\n", trget)
		os.Exit(1)
	}

	inPattern := false
	prevName := ""

	for tkn := range tknq {

		tag := tkn.Tag
		name := tkn.Name
		attr := tkn.Attr

		switch tag {
		case STARTTAG:
			prevName = name
			if name == pttrn {
				inPattern = true
				if which == CONTAINERTAG && what == DOREMOVE {
					continue
				}
			}
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			buffer.WriteString("<")
			buffer.WriteString(name)
			if attr != "" {
				if which != ATTRIBTAG || what != DOREMOVE {
					attr = strings.TrimSpace(attr)
					attr = CompressRunsOfSpaces(attr)
					buffer.WriteString(" ")
					buffer.WriteString(attr)
				}
			}
			buffer.WriteString(">\n")
		case SELFTAG:
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			buffer.WriteString("<")
			buffer.WriteString(name)
			if attr != "" {
				if which != ATTRIBTAG || what != DOREMOVE {
					attr = strings.TrimSpace(attr)
					attr = CompressRunsOfSpaces(attr)
					buffer.WriteString(" ")
					buffer.WriteString(attr)
				}
			}
			buffer.WriteString("/>\n")
		case STOPTAG:
			if name == pttrn {
				inPattern = false
				if which == OBJECTTAG && what == DOREMOVE {
					continue
				}
				if which == CONTAINERTAG && what == DOREMOVE {
					continue
				}
			}
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			buffer.WriteString("</")
			buffer.WriteString(name)
			buffer.WriteString(">\n")
		case CONTENTTAG:
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			if inPattern && which == CONTENTTAG && what == DOEXPAND {
				var words []string
				if strings.Contains(name, "|") {
					words = strings.FieldsFunc(name, func(c rune) bool {
						return c == '|'
					})
				} else if strings.Contains(name, ",") {
					words = strings.FieldsFunc(name, func(c rune) bool {
						return c == ','
					})
				} else {
					words = strings.Fields(name)
				}
				between := ""
				for _, item := range words {
					max := len(item)
					for max > 1 {
						ch := item[max-1]
						if ch != '.' && ch != ',' && ch != ':' && ch != ';' {
							break
						}
						// trim trailing punctuation
						item = item[:max-1]
						// continue checking for runs of punctuation at end
						max--
					}
					if HasFlankingSpace(item) {
						item = strings.TrimSpace(item)
					}
					if item != "" {
						if between != "" {
							buffer.WriteString(between)
						}
						buffer.WriteString(item)
						buffer.WriteString("\n")
						between = "</" + prevName + ">\n<" + prevName + ">\n"
					}
				}
				continue
			}
			if inPattern && which == tag {
				switch what {
				case DORETAIN:
					// default behavior for content - can use -filter X retain content as a no-op
				case DOREMOVE:
					continue
				case DOENCODE:
					name = html.EscapeString(name)
				case DODECODE:
					name = html.UnescapeString(name)
				case DOSHRINK:
					name = CompressRunsOfSpaces(name)
				case DOACCENT:
					if IsNotASCII(name) {
						name = DoAccentTransform(name)
					}
				default:
					continue
				}
			}
			// content normally printed
			if HasFlankingSpace(name) {
				name = strings.TrimSpace(name)
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case CDATATAG:
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			if inPattern && which == tag {
				switch what {
				case DORETAIN:
					// cdata requires explicit retain command
				case DOREMOVE:
					continue
				case DOENCODE:
					name = html.EscapeString(name)
				case DODECODE:
					name = html.UnescapeString(name)
				case DOSHRINK:
					name = CompressRunsOfSpaces(name)
				case DOACCENT:
					if IsNotASCII(name) {
						name = DoAccentTransform(name)
					}
				default:
					continue
				}
				// cdata normally removed
				if HasFlankingSpace(name) {
					name = strings.TrimSpace(name)
				}
				buffer.WriteString(name)
				buffer.WriteString("\n")
			}
		case COMMENTTAG:
			if inPattern && which == OBJECTTAG && what == DOREMOVE {
				continue
			}
			if inPattern && which == tag {
				switch what {
				case DORETAIN:
					// comment requires explicit retain command
				case DOREMOVE:
					continue
				case DOENCODE:
					name = html.EscapeString(name)
				case DODECODE:
					name = html.UnescapeString(name)
				case DOSHRINK:
					name = CompressRunsOfSpaces(name)
				case DOACCENT:
					if IsNotASCII(name) {
						name = DoAccentTransform(name)
					}
				default:
					continue
				}
				// comment normally removed
				if HasFlankingSpace(name) {
					name = strings.TrimSpace(name)
				}
				buffer.WriteString(name)
				buffer.WriteString("\n")
			}
		case DOCTYPETAG:
		case NOTAG:
		case ISCLOSED:
			txt := buffer.String()
			if txt != "" {
				// print final buffer
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			return
		default:
		}

		count++
		if count > 1000 {
			count = 0
			txt := buffer.String()
			if txt != "" {
				// print current buffered output
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			buffer.Reset()
		}
	}
}

// ProcessNormalize adjusts XML fields to conform to common conventions
func ProcessNormalize(rdr <-chan string, args []string) {

	if rdr == nil || args == nil {
		return
	}

	tknq := CreateTokenizer(rdr)

	if tknq == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create normalize tokenizer\n")
		os.Exit(1)
	}

	var buffer strings.Builder

	count := 0

	// e.g., transmute -normalize pubmed

	// skip past command name
	args = args[1:]

	max := len(args)
	if max < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: No database name supplied to transmute -normalize\n")
		os.Exit(1)
	}

	db := args[0]

	// force -strict cleanup flag
	switch db {
	case "bioc", "biocollections", "clinvar", "dbvar", "gap", "gapplus", "grasp", "pccompound", "pcsubstance":
	default:
		DoStrict = true
	}
	AllowEmbed = true
	ContentMods = true

	uid := ""
	isDocsum := false
	prevName := ""

	// removes mixed content tags
	mfix := strings.NewReplacer(
		"<b>", "",
		"<i>", "",
		"<u>", "",
		"</b>", "",
		"</i>", "",
		"</u>", "",
		"<b/>", "",
		"<i/>", "",
		"<u/>", "",
		"<sup>", "",
		"<sub>", "",
		"</sup>", "",
		"</sub>", "",
		"<sup/>", "",
		"<sub/>", "",
	)

	// reencodes < and > to &lt and &gt
	rfix := strings.NewReplacer(
		"<", "&lt;",
		">", "&gt;",
	)

	for tkn := range tknq {

		tag := tkn.Tag
		name := tkn.Name
		attr := tkn.Attr

		switch tag {
		case STARTTAG:
			if name == "Id" && uid != "" {
				uid = ""
			}
			if uid != "" {
				// if object after DocumentSummary is not already Id, create Id from rescued attribute
				buffer.WriteString("<Id>\n")
				buffer.WriteString(uid)
				buffer.WriteString("</Id>\n")
				// clear until next docsum
				uid = ""
			}
			if name == "DocumentSummary" {
				isDocsum = true
				if strings.HasPrefix(attr, "uid=\"") && strings.HasSuffix(attr, "\"") {
					attr = strings.TrimPrefix(attr, "uid=\"")
					attr = strings.TrimSuffix(attr, "\"")
					// store uid from DocumentSummary
					uid = attr
					// remove attribute
					attr = ""
				}
			}
			buffer.WriteString("<")
			buffer.WriteString(name)
			if attr != "" {
				attr = strings.TrimSpace(attr)
				attr = CompressRunsOfSpaces(attr)
				buffer.WriteString(" ")
				buffer.WriteString(attr)
			}
			buffer.WriteString(">\n")
			prevName = name
		case SELFTAG:
			buffer.WriteString("<")
			buffer.WriteString(name)
			if attr != "" {
				attr = strings.TrimSpace(attr)
				attr = CompressRunsOfSpaces(attr)
				buffer.WriteString(" ")
				buffer.WriteString(attr)
			}
			buffer.WriteString("/>\n")
		case STOPTAG:
			buffer.WriteString("</")
			buffer.WriteString(name)
			buffer.WriteString(">\n")
		case CONTENTTAG:
			if isDocsum {
				if db == "pubmed" && prevName == "Title" {
					if strings.Contains(name, "&") ||
						strings.Contains(name, "<") ||
						strings.Contains(name, ">") {
						ctype := tkn.Cont
						name = CleanupContents(name, (ctype&ASCII) != 0, (ctype&AMPER) != 0, (ctype&MIXED) != 0)
						if HasFlankingSpace(name) {
							name = strings.TrimSpace(name)
						}
						name = html.UnescapeString(name)
						// remove mixed content tags
						name = mfix.Replace(name)
						// reencode < and > to avoid breaking XML
						if strings.Contains(name, "<") || strings.Contains(name, ">") {
							name = rfix.Replace(name)
						}
					}
				} else if db == "gene" && prevName == "Summary" {
					if strings.Contains(name, "&amp;") {
						if HasFlankingSpace(name) {
							name = strings.TrimSpace(name)
						}
						name = html.UnescapeString(name)
						// reencode < and > to avoid breaking XML
						if strings.Contains(name, "<") || strings.Contains(name, ">") {
							name = rfix.Replace(name)
						}
					}
				} else if (db == "biosample" && prevName == "SampleData") ||
					(db == "medgen" && prevName == "ConceptMeta") ||
					(db == "sra" && prevName == "ExpXml") ||
					(db == "sra" && prevName == "Runs") {
					if strings.Contains(name, "&lt;") && strings.Contains(name, "&gt;") {
						if HasFlankingSpace(name) {
							name = strings.TrimSpace(name)
						}
						name = html.UnescapeString(name)
					}
				}
			} else {
				if db == "pubmed" {
					ctype := tkn.Cont
					name = CleanupContents(name, (ctype&ASCII) != 0, (ctype&AMPER) != 0, (ctype&MIXED) != 0)
					if HasFlankingSpace(name) {
						name = strings.TrimSpace(name)
					}
				} else if db == "bioc" {
					name = CleanupContents(name, true, true, true)
					if HasFlankingSpace(name) {
						name = strings.TrimSpace(name)
					}
				}
			}
			// content normally printed
			if HasFlankingSpace(name) {
				name = strings.TrimSpace(name)
			}
			buffer.WriteString(name)
			buffer.WriteString("\n")
		case CDATATAG:
			if isDocsum {
				if db == "assembly" && prevName == "Meta" {
					if strings.Contains(name, "<") && strings.Contains(name, ">") {
						// if CDATA contains embedded XML, simply remove CDATA wrapper
						if HasFlankingSpace(name) {
							name = strings.TrimSpace(name)
						}
						buffer.WriteString(name)
						buffer.WriteString("\n")
					}
				} else if db == "gtr" && prevName == "Extra" {
					// remove entire CDATA contents
				}
			}
		case COMMENTTAG:
			if !isDocsum {
				if db == "sra" {
					if strings.Contains(name, "<") && strings.Contains(name, ">") {
						// if comment contains embedded XML, remove comment wrapper and trim to leading < bracket
						pos := strings.Index(name, "<")
						if pos > 0 {
							name = name[pos:]
						}
						if HasFlankingSpace(name) {
							name = strings.TrimSpace(name)
						}
						buffer.WriteString(name)
						buffer.WriteString("\n")
					}
				}
			}
		case DOCTYPETAG:
			doctype := strings.TrimSpace(name)
			if strings.HasPrefix(doctype, "<") {
				doctype = doctype[1:]
			}
			if strings.HasPrefix(doctype, "!") {
				doctype = doctype[1:]
			}
			if strings.HasPrefix(doctype, "DOCTYPE") {
				doctype = doctype[7:]
			}
			if strings.HasPrefix(doctype, " ") {
				doctype = doctype[1:]
			}
			doctype = strings.TrimSuffix(doctype, ">")
			doctype = strings.TrimSpace(doctype)

			buffer.WriteString("<!DOCTYPE ")
			buffer.WriteString(doctype)
			buffer.WriteString(">")
		case NOTAG:
		case ISCLOSED:
			txt := buffer.String()
			if txt != "" {
				// print final buffer
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			return
		default:
		}

		count++
		if count > 1000 {
			count = 0
			txt := buffer.String()
			if txt != "" {
				// print current buffered output
				fmt.Fprintf(os.Stdout, "%s", txt)
			}
			buffer.Reset()
		}
	}
}

// JSONTokenizer sends sequential JSON tokens down a channel
func JSONTokenizer(inp io.Reader) <-chan string {

	if inp == nil {
		return nil
	}

	out := make(chan string, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create JSON tokenizer channel\n")
		os.Exit(1)
	}

	tokenizeJSON := func(inp io.Reader, out chan<- string) {

		// close channel when all tokens have been sent
		defer close(out)

		// use token decoder from encoding/json package
		dec := json.NewDecoder(inp)
		if dec == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create JSON Decoder\n")
			os.Exit(1)
		}
		dec.UseNumber()

		for {
			t, err := dec.Token()
			if err == io.EOF {
				return
			}
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to read JSON token '%s'\n", err)
				os.Exit(1)
			}

			// type switch performs sequential type assertions until match is found
			switch v := t.(type) {
			case json.Delim:
				// opening or closing braces (for objects) or brackets (for arrays)
				out <- string(v)
			case string:
				str := v
				if HasAdjacentSpacesOrNewline(str) {
					str = CompressRunsOfSpaces(str)
				}
				out <- str
			case json.Number:
				out <- v.String()
			case float64:
				out <- strconv.FormatFloat(v, 'f', -1, 64)
			case bool:
				if v {
					out <- "true"
				} else {
					out <- "false"
				}
			case nil:
				out <- "null"
			default:
				out <- t.(string)
			}
		}
	}

	// launch single tokenizer goroutine
	go tokenizeJSON(inp, out)

	return out
}

// JSONConverter parses JSON token stream into XML object stream
func JSONConverter(inp <-chan string, set, rec, nest string) <-chan string {

	if inp == nil {
		return nil
	}

	out := make(chan string, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create JSON converter channel\n")
		os.Exit(1)
	}

	// opt is used for anonymous top-level objects, anon for anonymous top-level arrays
	opt := "opt"
	anon := "anon"
	if rec != "" {
		// override record delimiter
		opt = rec
		anon = rec
	}

	flat := false
	plural := false
	depth := false

	switch nest {
	case "flat":
		flat = true
	case "plural", "name":
		plural = true
	case "depth", "deep", "level":
		depth = true
	}

	// convertJSON sends XML records down a channel
	convertJSON := func(inp <-chan string, out chan<- string) {

		// close channel when all tokens have been processed
		defer close(out)

		// ensure that XML tags are legal (initial digit allowed by xtract for biological data in JSON)
		fixTag := func(tag string) string {

			if tag == "" {
				return tag
			}

			okay := true
			for _, ch := range tag {
				if !InElement[ch] {
					okay = false
				}
			}
			if okay {
				return tag
			}

			var temp strings.Builder

			// replace illegal characters with underscore
			for _, ch := range tag {
				if InElement[ch] {
					temp.WriteRune(ch)
				} else {
					temp.WriteRune('_')
				}
			}

			return temp.String()
		}

		// closure silently places local variable pointer onto inner function call stack
		var buffer strings.Builder

		// array to speed up indentation
		indentSpaces := []string{
			"",
			"  ",
			"    ",
			"      ",
			"        ",
			"          ",
			"            ",
			"              ",
			"                ",
			"                  ",
		}

		indent := 0
		if set != "" {
			indent = 1
		}

		// indent a specified number of spaces
		doIndent := func(indt int) {
			i := indt
			for i > 9 {
				buffer.WriteString("                    ")
				i -= 10
			}
			if i < 0 {
				return
			}
			buffer.WriteString(indentSpaces[i])
		}

		count := 0

		// recursive function definitions
		var parseObject func(tag string)
		var parseArray func(tag, pfx string, lvl int)

		// recursive descent parser uses mutual recursion
		parseValue := func(tag, pfx, tkn string, lvl int) {

			switch tkn {
			case "{":
				parseObject(tag)
				// no break needed, would use fallthrough to explicitly cause program control to flow to the next case
			case "[":
				if flat {
					parseArray(tag, pfx, lvl+1)
				} else if lvl > 0 {
					// nested JSON arrays create recursive XML objects
					doIndent(indent)
					indent++
					tg := tag
					if plural {
						tg = inflector.Pluralize(tag)
					}
					buffer.WriteString("<")
					buffer.WriteString(tg)
					buffer.WriteString(">\n")
					if depth {
						parseArray(pfx+"_"+strconv.Itoa(lvl), tag, lvl+1)
					} else {
						parseArray(tag, pfx, lvl+1)
					}
					indent--
					doIndent(indent)
					buffer.WriteString("</")
					buffer.WriteString(tg)
					buffer.WriteString(">\n")
				} else {
					parseArray(tag, pfx, lvl+1)
				}
			case "}", "]":
				// should not get here, decoder tracks nesting of braces and brackets
			case "":
				// empty value string generates self-closing object
				doIndent(indent)
				buffer.WriteString("<")
				buffer.WriteString(tag)
				buffer.WriteString("/>\n")
			default:
				// write object and contents to string builder
				doIndent(indent)
				tkn = strings.TrimSpace(tkn)
				tkn = html.EscapeString(tkn)
				buffer.WriteString("<")
				buffer.WriteString(tag)
				buffer.WriteString(">")
				buffer.WriteString(tkn)
				buffer.WriteString("</")
				buffer.WriteString(tag)
				buffer.WriteString(">\n")
			}

			count++
			if count > 1000 {
				count = 0
				txt := buffer.String()
				if txt != "" {
					// send current result through output channel
					out <- txt
				}
				buffer.Reset()
			}
		}

		parseObject = func(tag string) {

			doIndent(indent)
			indent++
			buffer.WriteString("<")
			buffer.WriteString(tag)
			buffer.WriteString(">\n")

			for {
				// shadowing tag variable inside for loop does not step on value of tag argument in outer scope
				tag, ok := <-inp
				if !ok {
					break
				}

				if tag == "}" || tag == "]" {
					break
				}

				tag = fixTag(tag)

				tkn, ok := <-inp
				if !ok {
					break
				}

				if tkn == "}" || tkn == "]" {
					break
				}

				parseValue(tag, tag, tkn, 0)
			}

			indent--
			doIndent(indent)
			buffer.WriteString("</")
			buffer.WriteString(tag)
			buffer.WriteString(">\n")
		}

		parseArray = func(tag, pfx string, lvl int) {

			for {
				tkn, ok := <-inp
				if !ok {
					break
				}

				if tkn == "}" || tkn == "]" {
					break
				}

				parseValue(tag, pfx, tkn, lvl)
			}
		}

		// process stream of catenated top-level JSON objects or arrays
		for {
			tkn, ok := <-inp
			if !ok {
				break
			}
			if tkn == "{" {
				parseObject(opt)
			} else if tkn == "[" {
				parseArray(anon, anon, 0)
			} else {
				break
			}

			txt := buffer.String()
			if txt != "" {
				// send remaining result through output channel
				out <- txt
			}

			buffer.Reset()

			runtime.Gosched()
		}
	}

	// launch single converter goroutine
	go convertJSON(inp, out)

	return out
}

// ASN1Tokenizer sends sequential ASN1 tokens down a channel
func ASN1Tokenizer(inp io.Reader) <-chan string {

	if inp == nil {
		return nil
	}

	out := make(chan string, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create ASN1 tokenizer channel\n")
		os.Exit(1)
	}

	var buf strings.Builder

	scanr := bufio.NewScanner(inp)

	tokenizeASN1 := func(inp io.Reader, out chan<- string) {

		// close channel when all tokens have been sent
		defer close(out)

		row := 0
		idx := 0
		line := ""

		sentinel := string(rune(0))

		nextLine := func() string {

			for scanr.Scan() {
				// read line
				line := scanr.Text()
				row++
				if line == "" {
					// ignore blank lines
					continue
				}
				// add sentinel
				line += sentinel
				return line
			}

			// end of data
			return sentinel
		}

		readRestOfAsnString := func() string {

			// continue reading additional lines of string
			for {

				line = nextLine()
				if line == "" || line == sentinel {
					break
				}
				idx = 0

				ch := line[idx]

				for InAsnString[ch] {
					idx++
					ch = line[idx]
				}
				str := line[:idx]
				line = line[idx:]
				idx = 0

				buf.WriteString(str)

				if strings.HasPrefix(line, "\"") {
					// "
					// skip past closing quote
					line = line[1:]
					idx = 0

					// break out of continuation loop
					return line
				}

				// string continues on additional lines
			}

			return line
		}

		readRestOfAsnBits := func() string {

			// continue reading additional lines of string
			for {

				line = nextLine()
				if line == "" || line == sentinel {
					break
				}
				idx = 0

				ch := line[idx]

				for InAsnBits[ch] {
					idx++
					ch = line[idx]
				}
				str := line[:idx]
				line = line[idx:]
				idx = 0

				buf.WriteString(str)

				if strings.HasPrefix(line, "'") {
					// skip past closing apostrophe
					line = line[1:]
					idx = 0

					// break out of continuation loop
					return line
				}

				// string continues on additional lines
			}

			return line
		}

		for {

			line = nextLine()
			if line == "" || line == sentinel {
				break
			}

			for {

				if line == "" || line == sentinel {
					break
				}
				idx = 0

				// trim leading blanks
				ch := line[idx]
				for InBlank[ch] {
					idx++
					ch = line[idx]
				}
				line = line[idx:]
				idx = 0

				if ch == ',' {
					out <- string(ch)
					line = line[1:]
					continue
				}

				if ch == '{' {
					// start structure
					out <- string(ch)
					line = line[1:]
					continue
				}

				if ch == '}' {
					// end structure
					out <- string(ch)
					line = line[1:]
					continue
				}

				if ch == '"' {
					// "
					// start of string
					buf.Reset()

					// skip past opening quote
					line = line[1:]
					idx = 0
					ch = line[idx]

					// read to closing quote or sentinel at end of line
					for InAsnString[ch] {
						idx++
						ch = line[idx]
					}
					str := line[:idx]
					line = line[idx:]
					idx = 0

					buf.WriteString(str)

					if strings.HasPrefix(line, "\"") {
						// "
						// skip past closing quote
						line = line[1:]
						idx = 0

					} else {

						// continue reading additional lines of string
						line = readRestOfAsnString()
						idx = 0
					}

					tmp := buf.String()
					if tmp == "" {
						// encode empty string
						tmp = "\"\""
					}

					out <- tmp
					buf.Reset()

					continue
				}

				if ch == '\'' {
					// start of bit string
					buf.Reset()

					// skip past opening apostrophe
					line = line[1:]
					idx = 0
					ch = line[idx]

					// read to closing apostrophe or sentinel at end of line
					for InAsnBits[ch] {
						idx++
						ch = line[idx]
					}
					str := line[:idx]
					line = line[idx:]
					idx = 0

					buf.WriteString(str)

					if strings.HasPrefix(line, "'") {
						// skip past closing apostrophe
						line = line[1:]
						idx = 0

					} else {

						// continue reading additional lines of bit string
						line = readRestOfAsnBits()
						idx = 0
					}

					// if apostrophe is at end of line, read next line
					if line == "" || line == sentinel {
						line = nextLine()
					}

					// then skip past trailing hex or binary indicator
					if strings.HasPrefix(line, "H") || strings.HasPrefix(line, "B") {
						line = line[1:]
						idx = 0
					}

					out <- buf.String()
					buf.Reset()

					continue
				}

				if ch == ':' && strings.HasPrefix(line, "::=") {
					// start of record contents
					out <- "::="
					line = line[3:]
					idx = 0
					continue
				}

				if ch == '-' && strings.HasPrefix(line, "--") {
					// skip comments
					break
				}
				if ch == ';' {
					// skip comments
					break
				}

				// read token or unquoted numeric value
				idx = 0
				for InAsnTag[ch] {
					idx++
					ch = line[idx]
				}
				tkn := line[:idx]
				line = line[idx:]
				idx = 0

				out <- tkn
			}
		}
	}

	// launch single tokenizer goroutine
	go tokenizeASN1(inp, out)

	return out
}

// ASN1Converter parses ASN1 token stream into XML object stream
func ASN1Converter(inp <-chan string, set, rec string) <-chan string {

	if inp == nil {
		return nil
	}

	out := make(chan string, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create ASN1 converter channel\n")
		os.Exit(1)
	}

	// convertASN1 sends XML records down a channel
	convertASN1 := func(inp <-chan string, out chan<- string) {

		// close channel when all tokens have been processed
		defer close(out)

		// ensure that XML tags are legal
		fixTag := func(tag string) string {

			if tag == "" {
				return tag
			}

			okay := true
			for _, ch := range tag {
				if !InElement[ch] {
					okay = false
				}
			}
			if okay {
				return tag
			}

			var temp strings.Builder

			// replace illegal characters with underscore
			for _, ch := range tag {
				if InElement[ch] {
					temp.WriteRune(ch)
				} else {
					temp.WriteRune('_')
				}
			}

			return temp.String()
		}

		nextToken := func() string {

			for {
				tkn, ok := <-inp
				if !ok {
					break
				}
				if tkn == "" {
					// ignore blank tokens
					continue
				}

				return tkn
			}

			// end of data
			return ""
		}

		// collects tags until next brace or comma
		var arry []string

		// builds XML output for current record
		var buffer strings.Builder

		// array to speed up indentation
		indentSpaces := []string{
			"",
			"  ",
			"    ",
			"      ",
			"        ",
			"          ",
			"            ",
			"              ",
			"                ",
			"                  ",
		}

		count := 0

		indent := 0
		if set != "" {
			indent = 1
		}

		// indent a specified number of spaces
		doIndent := func(indt int) {
			i := indt
			for i > 9 {
				buffer.WriteString("                    ")
				i -= 10
			}
			if i < 0 {
				return
			}
			buffer.WriteString(indentSpaces[i])
		}

		printOpeningTag := func(tag string) {

			tag = fixTag(tag)
			doIndent(indent)
			indent++
			buffer.WriteString("<")
			buffer.WriteString(tag)
			buffer.WriteString(">\n")
		}

		printClosingTag := func(tag string) {

			tag = fixTag(tag)
			indent--
			doIndent(indent)
			buffer.WriteString("</")
			buffer.WriteString(tag)
			buffer.WriteString(">\n")
		}

		printContent := func(tag, tkn string) {

			if tkn == "\"\"" {
				return
			}
			tag = fixTag(tag)
			doIndent(indent)
			buffer.WriteString("<")
			buffer.WriteString(tag)
			buffer.WriteString(">")
			tkn = strings.TrimSpace(tkn)
			tkn = html.EscapeString(tkn)
			buffer.WriteString(tkn)
			buffer.WriteString("</")
			buffer.WriteString(tag)
			buffer.WriteString(">\n")
		}

		popFromArry := func() (string, string, string) {

			fst, sec, trd := "", "", ""
			switch len(arry) {
			case 1:
				fst = arry[0]
			case 2:
				fst, sec = arry[0], arry[1]
			case 3:
				fst, sec, trd = arry[0], arry[1], arry[2]
			}
			arry = nil
			return fst, sec, trd
		}

		// recursive function definition
		var parseAsnObject func(prnt string, lvl int)

		parseAsnObject = func(prnt string, lvl int) {

			for {
				tkn := nextToken()
				if tkn == "" {
					return
				}

				switch tkn {
				case "{":
					fst, sec, trd := popFromArry()
					tag := fst
					if fst == "" {
						fst = prnt
					}
					if fst == "" {
						return
					}
					printOpeningTag(fst)
					tag = fst + "_E"
					if sec != "" {
						printOpeningTag(sec)
						tag = sec + "_E"
					}
					if trd != "" {
						printOpeningTag(trd)
						tag = trd
					}
					parseAsnObject(tag, lvl+1)
					if trd != "" {
						printClosingTag(trd)
					}
					if sec != "" {
						printClosingTag(sec)
					}
					printClosingTag(fst)
					if lvl == 0 {
						return
					}
				case ",":
					fst, sec, trd := popFromArry()
					if trd != "" {
						printOpeningTag(fst)
						printContent(sec, trd)
						printClosingTag(fst)
					} else if sec != "" {
						printContent(fst, sec)
					} else if fst != "" {
						printContent(prnt, fst)
					}
				case "}":
					fst, sec, trd := popFromArry()
					if fst == "" {
						return
					}
					if trd != "" {
						printOpeningTag(fst)
						printContent(sec, trd)
						printClosingTag(fst)
					} else if sec != "" {
						printContent(fst, sec)
					} else if fst != "" {
						printContent(prnt, fst)
					}
					return
				case "::=":
					fmt.Fprintf(os.Stderr, "\nERROR: Unexpected ::= token found\n")
					os.Exit(1)
				default:
					arry = append(arry, tkn)
				}

				count++
				if count > 1000 {
					count = 0
					txt := buffer.String()
					if txt != "" {
						// send current result through output channel
						out <- txt
					}
					buffer.Reset()
				}
			}
		}

		if set != "" {
			out <- "<" + set + ">"
		}

		// process stream of catenated top-level ASN1 records
		for {
			arry = nil

			top := nextToken()
			if top == "" {
				break
			}

			if rec != "" {
				top = rec
			}

			arry = append(arry, top)

			tkn := nextToken()
			if tkn == "" {
				fmt.Fprintf(os.Stderr, "\nERROR: Incomplete ASN1 starting with '%s'\n", top)
				os.Exit(1)
			}
			if tkn != "::=" {
				fmt.Fprintf(os.Stderr, "\nERROR: ASN1 message missing expected ::= token, found '%s'\n", tkn)
				os.Exit(1)
			}

			parseAsnObject(top, 0)

			txt := buffer.String()
			if txt != "" {
				// send remaining result through output channel
				out <- txt
			}

			buffer.Reset()

			runtime.Gosched()
		}

		if set != "" {
			out <- "</" + set + ">"
		}
	}

	// launch single converter goroutine
	go convertASN1(inp, out)

	return out
}

// READ TAB-DELIMITED FILE AND WRAP IN XML FIELDS

// TableConverter parses tab-delimited (-t2x) or comma-separated values (-c2x) files into XML
func TableConverter(inp io.Reader, delim string, args []string) int {

	// e.g., transmute -t2x -set Set -rec Rec -skip 0 Uid Name

	if inp == nil {
		return 0
	}

	cmmd := args[0]
	args = args[1:]

	head := ""
	tail := ""

	hd := ""
	tl := ""

	skip := 0
	lower := false
	upper := false
	indent := true

	var fields []string
	numFlds := 0

	for len(args) > 0 {
		str := args[0]
		switch str {
		case "-set":
			args = args[1:]
			if len(args) < 1 {
				fmt.Fprintf(os.Stderr, "\nERROR: No argument after -set\n")
				os.Exit(1)
			}
			set := args[0]
			if set != "" && set != "-" {
				head = "<" + set + ">"
				tail = "</" + set + ">"
			}
			args = args[1:]
		case "-rec":
			args = args[1:]
			if len(args) < 1 {
				fmt.Fprintf(os.Stderr, "\nERROR: No argument after -rec\n")
				os.Exit(1)
			}
			rec := args[0]
			if rec != "" && rec != "-" {
				hd = "<" + rec + ">"
				tl = "</" + rec + ">"
			}
			args = args[1:]
		case "-skip":
			args = args[1:]
			if len(args) < 1 {
				fmt.Fprintf(os.Stderr, "\nERROR: No argument after -skip\n")
				os.Exit(1)
			}
			tmp := args[0]
			val, err := strconv.Atoi(tmp)
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: -skip argument (%s) is not an integer\n", tmp)
				os.Exit(1)
			}
			skip = val
			args = args[1:]
		case "-lower":
			lower = true
			args = args[1:]
		case "-upper":
			upper = true
			args = args[1:]
		case "-indent":
			indent = true
			args = args[1:]
		case "-flush":
			indent = false
			args = args[1:]
		default:
			// remaining arguments are names for columns
			if str != "" && str != "*" {
				fields = append(fields, str)
				numFlds++
			}
			args = args[1:]
		}
	}

	if numFlds < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Insufficient arguments for %s\n", cmmd)
		os.Exit(1)
	}

	var buffer strings.Builder
	count := 0
	okay := false
	row := 0
	recordCount := 0

	wrtr := bufio.NewWriter(os.Stdout)

	scanr := bufio.NewScanner(inp)

	if head != "" {
		buffer.WriteString(head)
		buffer.WriteString("\n")
	}

	for scanr.Scan() {

		line := scanr.Text()

		row++

		if skip > 0 {
			skip--
			continue
		}

		cols := strings.Split(line, delim)

		if len(cols) != numFlds {
			fmt.Fprintf(os.Stderr, "Mismatched columns in row %d - '%s'\n", row, line)
			continue
		}

		if hd != "" {
			if indent {
				buffer.WriteString("  ")
			}
			buffer.WriteString(hd)
			buffer.WriteString("\n")
		}

		for i, fld := range fields {
			val := cols[i]
			if lower {
				val = strings.ToLower(val)
			}
			if upper {
				val = strings.ToUpper(val)
			}
			if fld[0] == '*' {
				fld = fld[1:]
			} else {
				val = html.EscapeString(val)
			}
			val = strings.TrimSpace(val)
			if indent {
				buffer.WriteString("    ")
			}
			buffer.WriteString("<")
			buffer.WriteString(fld)
			buffer.WriteString(">")
			buffer.WriteString(val)
			buffer.WriteString("</")
			buffer.WriteString(fld)
			buffer.WriteString(">")
			buffer.WriteString("\n")
		}

		if tl != "" {
			if indent {
				buffer.WriteString("  ")
			}
			buffer.WriteString(tl)
			buffer.WriteString("\n")
		}

		recordCount++
		count++

		if count >= 1000 {
			count = 0
			txt := buffer.String()
			if txt != "" {
				// print current buffer
				wrtr.WriteString(txt[:])
			}
			buffer.Reset()
		}

		okay = true
	}

	if tail != "" {
		buffer.WriteString(tail)
		buffer.WriteString("\n")
	}

	if okay {
		txt := buffer.String()
		if txt != "" {
			// print current buffer
			wrtr.WriteString(txt[:])
		}
	}
	buffer.Reset()

	wrtr.Flush()

	return recordCount
}

// READ GENBANK FLATFILE AND TRANSLATE TO INSDSEQ XML

// GenBankConverter sends INSDSeq XML records down a channel
func GenBankConverter(inp io.Reader) <-chan string {

	if inp == nil {
		return nil
	}

	out := make(chan string, ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "Unable to create GenBank converter channel\n")
		os.Exit(1)
	}

	const twelvespaces = "            "
	const twentyonespaces = "                     "

	var rec strings.Builder
	var alt strings.Builder
	var con strings.Builder
	var seq strings.Builder

	scanr := bufio.NewScanner(inp)

	convertGenBank := func(inp io.Reader, out chan<- string) {

		// close channel when all records have been sent
		defer close(out)

		row := 0

		nextLine := func() string {

			for scanr.Scan() {
				line := scanr.Text()
				if line == "" {
					continue
				}
				return line
			}
			return ""

		}

		for {

			rec.Reset()

			// read first line of next record
			line := nextLine()
			if line == "" {
				break
			}

			row++

			for {
				if !strings.HasPrefix(line, "LOCUS") {
					// skip release file header information
					line = nextLine()
					row++
					continue
				}
				break
			}

			readContinuationLines := func(str string) string {

				for {
					// read next line
					line = nextLine()
					row++
					if !strings.HasPrefix(line, twelvespaces) {
						// if not continuation line, break out of loop
						break
					}
					// append subsequent line and continue with loop
					txt := strings.TrimPrefix(line, twelvespaces)
					str += " " + txt
				}

				str = CompressRunsOfSpaces(str)
				str = strings.TrimSpace(str)

				return str
			}

			writeOneElement := func(spaces, tag, value string) {

				rec.WriteString(spaces)
				rec.WriteString("<")
				rec.WriteString(tag)
				rec.WriteString(">")
				value = html.EscapeString(value)
				rec.WriteString(value)
				rec.WriteString("</")
				rec.WriteString(tag)
				rec.WriteString(">\n")
			}

			// each section will exit with the next line ready to process

			if strings.HasPrefix(line, "LOCUS") {

				// start of record
				rec.WriteString("  <INSDSeq>\n")

				// do not break if given artificial multi-line LOCUS
				str := readContinuationLines(line)

				cols := strings.Fields(str)
				ln := len(cols)
				if ln == 8 {
					moleculetype := cols[4]
					strandedness := ""
					if strings.HasPrefix(moleculetype, "ds-") {
						moleculetype = strings.TrimPrefix(moleculetype, "ds-")
						strandedness = "double"
					} else if strings.HasPrefix(moleculetype, "ss-") {
						moleculetype = strings.TrimPrefix(moleculetype, "ss-")
						strandedness = "single"
					} else if strings.HasPrefix(moleculetype, "ms-") {
						moleculetype = strings.TrimPrefix(moleculetype, "ms-")
						strandedness = "mixed"
					} else if strings.HasSuffix(moleculetype, "DNA") {
						strandedness = "double"
					} else if strings.HasSuffix(moleculetype, "RNA") {
						strandedness = "single"
					}

					writeOneElement("    ", "INSDSeq_locus", cols[1])

					writeOneElement("    ", "INSDSeq_length", cols[2])

					if strandedness != "" {
						writeOneElement("    ", "INSDSeq_strandedness", strandedness)
					}

					writeOneElement("    ", "INSDSeq_moltype", moleculetype)

					writeOneElement("    ", "INSDSeq_topology", cols[5])

					writeOneElement("    ", "INSDSeq_division", cols[6])

					writeOneElement("    ", "INSDSeq_update-date", cols[7])

				} else if ln == 7 {

					writeOneElement("    ", "INSDSeq_locus", cols[1])

					writeOneElement("    ", "INSDSeq_length", cols[2])

					writeOneElement("    ", "INSDSeq_moltype", "AA")

					writeOneElement("    ", "INSDSeq_topology", cols[4])

					writeOneElement("    ", "INSDSeq_division", cols[5])

					writeOneElement("    ", "INSDSeq_update-date", cols[6])

				} else {
					fmt.Fprintf(os.Stderr, "ERROR: "+str+"\n")
				}

				// read next line and continue - handled by readContinuationLines above
				// line = nextLine()
				// row++
			}

			if strings.HasPrefix(line, "DEFINITION") {

				txt := strings.TrimPrefix(line, "DEFINITION")
				def := readContinuationLines(txt)
				def = strings.TrimSuffix(def, ".")

				writeOneElement("    ", "INSDSeq_definition", def)
			}

			var secondaries []string

			if strings.HasPrefix(line, "ACCESSION") {

				txt := strings.TrimPrefix(line, "ACCESSION")
				str := readContinuationLines(txt)
				accessions := strings.Fields(str)
				ln := len(accessions)
				if ln > 1 {

					writeOneElement("    ", "INSDSeq_primary-accession", accessions[0])

					// skip past primary accession, collect secondaries
					secondaries = accessions[1:]

				} else if ln == 1 {

					writeOneElement("    ", "INSDSeq_primary-accession", accessions[0])

				} else {
					fmt.Fprintf(os.Stderr, "ERROR: ACCESSION "+str+"\n")
				}
			}

			accnver := ""
			gi := ""

			if strings.HasPrefix(line, "VERSION") {

				cols := strings.Fields(line)
				if len(cols) == 2 {

					accnver = cols[1]
					writeOneElement("    ", "INSDSeq_accession-version", accnver)

				} else if len(cols) == 3 {

					accnver = cols[1]
					writeOneElement("    ", "INSDSeq_accession-version", accnver)

					// collect gi for other-seqids
					if strings.HasPrefix(cols[2], "GI:") {
						gi = strings.TrimPrefix(cols[2], "GI:")
					}

				} else {
					fmt.Fprintf(os.Stderr, "ERROR: "+line+"\n")
				}

				// read next line and continue
				line = nextLine()
				row++

			}

			if gi != "" {

				rec.WriteString("    <INSDSeq_other-seqids>\n")

				writeOneElement("      ", "INSDSeqid", "gi|"+gi)

				rec.WriteString("    </INSDSeq_other-seqids>\n")
			}

			if len(secondaries) > 0 {

				rec.WriteString("    <INSDSeq_secondary-accessions>\n")

				for _, secndry := range secondaries {

					if strings.HasPrefix(secndry, "REGION") {
						break
					}
					writeOneElement("      ", "INSDSecondary-accn", secndry)
				}

				rec.WriteString("    </INSDSeq_secondary-accessions>\n")
			}

			if strings.HasPrefix(line, "DBLINK") {

				txt := strings.TrimPrefix(line, "DBLINK")
				readContinuationLines(txt)
				// collect for database-reference
				// out <- Token{DBLINK, dbl}
			}

			if strings.HasPrefix(line, "DBSOURCE") {

				txt := strings.TrimPrefix(line, "DBSOURCE")
				readContinuationLines(txt)
				// collect for database-source
				// out <- Token{DBSOURCE, dbl}
			}

			if strings.HasPrefix(line, "KEYWORDS") {

				txt := strings.TrimPrefix(line, "KEYWORDS")
				key := readContinuationLines(txt)
				key = strings.TrimSuffix(key, ".")

				if key != "" {
					rec.WriteString("    <INSDSeq_keywords>\n")
					kywds := strings.Split(key, ";")
					for _, kw := range kywds {
						kw = strings.TrimSpace(kw)
						if kw == "" || kw == "." {
							continue
						}

						writeOneElement("      ", "INSDKeyword", kw)
					}
					rec.WriteString("    </INSDSeq_keywords>\n")
				}
			}

			if strings.HasPrefix(line, "SOURCE") {

				txt := strings.TrimPrefix(line, "SOURCE")
				src := readContinuationLines(txt)

				writeOneElement("    ", "INSDSeq_source", src)
			}

			if strings.HasPrefix(line, "  ORGANISM") {

				org := strings.TrimPrefix(line, "  ORGANISM")
				org = CompressRunsOfSpaces(org)
				org = strings.TrimSpace(org)

				writeOneElement("    ", "INSDSeq_organism", org)

				line = nextLine()
				row++
				if strings.HasPrefix(line, twelvespaces) {
					txt := strings.TrimPrefix(line, twelvespaces)
					tax := readContinuationLines(txt)
					tax = strings.TrimSuffix(tax, ".")

					writeOneElement("    ", "INSDSeq_taxonomy", tax)
				}
			}

			rec.WriteString("    <INSDSeq_references>\n")
			for {
				if !strings.HasPrefix(line, "REFERENCE") {
					// exit out of reference section
					break
				}

				ref := "0"

				rec.WriteString("      <INSDReference>\n")

				txt := strings.TrimPrefix(line, "REFERENCE")
				str := readContinuationLines(txt)
				str = CompressRunsOfSpaces(str)
				str = strings.TrimSpace(str)
				idx := strings.Index(str, "(")
				if idx > 0 {
					ref = strings.TrimSpace(str[:idx])

					writeOneElement("        ", "INSDReference_reference", ref)

					posn := str[idx+1:]
					posn = strings.TrimSuffix(posn, ")")
					posn = strings.TrimSpace(posn)
					if posn == "sites" {

						writeOneElement("        ", "INSDReference_position", posn)

					} else {
						var arry []string
						cls := strings.Split(posn, ";")
						for _, item := range cls {
							item = strings.TrimPrefix(item, "bases ")
							item = strings.TrimPrefix(item, "residues ")
							item = strings.TrimSpace(item)
							cols := strings.Fields(item)
							if len(cols) == 3 && cols[1] == "to" {
								arry = append(arry, cols[0]+".."+cols[2])
							}
						}
						if len(arry) > 0 {
							posit := strings.Join(arry, ",")
							writeOneElement("        ", "INSDReference_position", posit)
						} else {
							fmt.Fprintf(os.Stderr, "ERROR: "+posn+"\n")
						}
					}
				} else {
					ref = strings.TrimSpace(str)

					writeOneElement("        ", "INSDReference_reference", ref)
				}
				row++

				if strings.HasPrefix(line, "  AUTHORS") {

					txt := strings.TrimPrefix(line, "  AUTHORS")
					auths := readContinuationLines(txt)

					rec.WriteString("        <INSDReference_authors>\n")
					authors := strings.Split(auths, ", ")
					for _, auth := range authors {
						auth = strings.TrimSpace(auth)
						if auth == "" {
							continue
						}
						pair := strings.Split(auth, " and ")
						for _, name := range pair {

							writeOneElement("          ", "INSDAuthor", name)
						}
					}
					rec.WriteString("        </INSDReference_authors>\n")
				}

				if strings.HasPrefix(line, "  CONSRTM") {

					txt := strings.TrimPrefix(line, "  CONSRTM")
					cons := readContinuationLines(txt)

					writeOneElement("        ", "INSDReference_consortium", cons)
				}

				if strings.HasPrefix(line, "  TITLE") {

					txt := strings.TrimPrefix(line, "  TITLE")
					titl := readContinuationLines(txt)

					writeOneElement("        ", "INSDReference_title", titl)
				}

				if strings.HasPrefix(line, "  JOURNAL") {

					txt := strings.TrimPrefix(line, "  JOURNAL")
					jour := readContinuationLines(txt)

					writeOneElement("        ", "INSDReference_journal", jour)
				}

				if strings.HasPrefix(line, "   PUBMED") {

					txt := strings.TrimPrefix(line, "   PUBMED")
					pmid := readContinuationLines(txt)

					writeOneElement("        ", "INSDReference_pubmed", pmid)
				}

				if strings.HasPrefix(line, "  MEDLINE") {

					txt := strings.TrimPrefix(line, "  MEDLINE")
					// old MEDLINE uid not supported
					readContinuationLines(txt)
				}

				if strings.HasPrefix(line, "  REMARK") {

					txt := strings.TrimPrefix(line, "  REMARK")
					rem := readContinuationLines(txt)

					writeOneElement("        ", "INSDReference_remark", rem)
				}

				// end of this reference
				rec.WriteString("      </INSDReference>\n")
				// continue to next reference
			}
			rec.WriteString("    </INSDSeq_references>\n")

			if strings.HasPrefix(line, "COMMENT") {

				txt := strings.TrimPrefix(line, "COMMENT")
				com := readContinuationLines(txt)

				writeOneElement("    ", "INSDSeq_comment", com)
			}

			rec.WriteString("    <INSDSeq_feature-table>\n")
			if strings.HasPrefix(line, "FEATURES") {

				line = nextLine()
				row++

				for {
					if !strings.HasPrefix(line, "     ") {
						// exit out of features section
						break
					}
					if len(line) < 22 {
						fmt.Fprintf(os.Stderr, "ERROR: "+line+"\n")
						line = nextLine()
						row++
						continue
					}

					rec.WriteString("      <INSDFeature>\n")

					// read feature key and start of location
					fkey := line[5:21]
					fkey = strings.TrimSpace(fkey)

					writeOneElement("        ", "INSDFeature_key", fkey)

					loc := line[21:]
					loc = strings.TrimSpace(loc)
					for {
						line = nextLine()
						row++
						if !strings.HasPrefix(line, twentyonespaces) {
							break
						}
						txt := strings.TrimPrefix(line, twentyonespaces)
						if strings.HasPrefix(txt, "/") {
							// if not continuation of location, break out of loop
							break
						}
						// append subsequent line and continue with loop
						loc += strings.TrimSpace(txt)
					}

					writeOneElement("        ", "INSDFeature_location", loc)

					location_operator := ""
					is_comp := false
					prime5 := false
					prime3 := false

					// parseloc recursive definition
					var parseloc func(string) []string

					parseloc = func(str string) []string {

						var acc []string

						if strings.HasPrefix(str, "join(") && strings.HasSuffix(str, ")") {

							location_operator = "join"

							str = strings.TrimPrefix(str, "join(")
							str = strings.TrimSuffix(str, ")")
							items := strings.Split(str, ",")

							for _, thisloc := range items {
								inner := parseloc(thisloc)
								for _, sub := range inner {
									acc = append(acc, sub)
								}
							}

						} else if strings.HasPrefix(str, "order(") && strings.HasSuffix(str, ")") {

							location_operator = "order"

							str = strings.TrimPrefix(str, "order(")
							str = strings.TrimSuffix(str, ")")
							items := strings.Split(str, ",")

							for _, thisloc := range items {
								inner := parseloc(thisloc)
								for _, sub := range inner {
									acc = append(acc, sub)
								}
							}

						} else if strings.HasPrefix(str, "complement(") && strings.HasSuffix(str, ")") {

							is_comp = true

							str = strings.TrimPrefix(str, "complement(")
							str = strings.TrimSuffix(str, ")")
							items := parseloc(str)

							// reverse items
							for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 {
								items[i], items[j] = items[j], items[i]
							}

							// reverse from and to positions, flip direction of angle brackets (partial flags)
							for _, thisloc := range items {
								pts := strings.Split(thisloc, "..")
								ln := len(pts)
								if ln == 2 {
									fst := pts[0]
									scd := pts[1]
									lf := ""
									rt := ""
									if strings.HasPrefix(fst, "<") {
										fst = strings.TrimPrefix(fst, "<")
										rt = ">"
									}
									if strings.HasPrefix(scd, ">") {
										scd = strings.TrimPrefix(scd, ">")
										lf = "<"
									}
									acc = append(acc, lf+scd+".."+rt+fst)
								} else if ln > 0 {
									acc = append(acc, pts[0])
								}
							}

						} else {

							// save individual interval or point if no leading accession
							if strings.Index(str, ":") < 0 {
								acc = append(acc, str)
							}
						}

						return acc
					}

					items := parseloc(loc)

					rec.WriteString("        <INSDFeature_intervals>\n")

					num_ivals := 0

					// report individual intervals
					for _, thisloc := range items {
						if thisloc == "" {
							continue
						}

						num_ivals++

						rec.WriteString("          <INSDInterval>\n")
						pts := strings.Split(thisloc, "..")
						if len(pts) == 2 {

							// fr..to
							fr := pts[0]
							to := pts[1]
							if strings.HasPrefix(fr, "<") {
								fr = strings.TrimPrefix(fr, "<")
								prime5 = true
							}
							if strings.HasPrefix(to, ">") {
								to = strings.TrimPrefix(to, ">")
								prime3 = true
							}
							writeOneElement("            ", "INSDInterval_from", fr)
							writeOneElement("            ", "INSDInterval_to", to)
							if is_comp {
								rec.WriteString("            <INSDInterval_iscomp value=\"true\"/>\n")
							}
							writeOneElement("            ", "INSDInterval_accession", accnver)

						} else {

							crt := strings.Split(thisloc, "^")
							if len(crt) == 2 {

								// fr^to
								fr := crt[0]
								to := crt[1]
								writeOneElement("            ", "INSDInterval_from", fr)
								writeOneElement("            ", "INSDInterval_to", to)
								if is_comp {
									rec.WriteString("            <INSDInterval_iscomp value=\"true\"/>\n")
								}
								rec.WriteString("            <INSDInterval_interbp value=\"true\"/>\n")
								writeOneElement("            ", "INSDInterval_accession", accnver)

							} else {

								// pt
								pt := pts[0]
								if strings.HasPrefix(pt, "<") {
									pt = strings.TrimPrefix(pt, "<")
									prime5 = true
								}
								if strings.HasPrefix(pt, ">") {
									pt = strings.TrimPrefix(pt, ">")
									prime3 = true
								}
								writeOneElement("            ", "INSDInterval_point", pt)
								writeOneElement("            ", "INSDInterval_accession", accnver)
							}
						}
						rec.WriteString("          </INSDInterval>\n")
					}

					rec.WriteString("        </INSDFeature_intervals>\n")

					if num_ivals > 1 {
						writeOneElement("        ", "INSDFeature_operator", location_operator)
					}
					if prime5 {
						rec.WriteString("        <INSDFeature_partial5 value=\"true\"/>\n")
					}
					if prime3 {
						rec.WriteString("        <INSDFeature_partial3 value=\"true\"/>\n")
					}

					hasQual := false
					for {
						if !strings.HasPrefix(line, twentyonespaces) {
							// if not qualifier line, break out of loop
							break
						}
						txt := strings.TrimPrefix(line, twentyonespaces)
						qual := ""
						val := ""
						if strings.HasPrefix(txt, "/") {
							if !hasQual {
								hasQual = true
								rec.WriteString("        <INSDFeature_quals>\n")
							}
							// read new qualifier and start of value
							qual = strings.TrimPrefix(txt, "/")
							qual = strings.TrimSpace(qual)
							idx := strings.Index(qual, "=")
							if idx > 0 {
								val = qual[idx+1:]
								qual = qual[:idx]
							}

							for {
								line = nextLine()
								row++
								if !strings.HasPrefix(line, twentyonespaces) {
									break
								}
								txt := strings.TrimPrefix(line, twentyonespaces)
								if strings.HasPrefix(txt, "/") {
									// if not continuation of qualifier, break out of loop
									break
								}
								// append subsequent line to value and continue with loop
								if qual == "transcription" || qual == "translation" || qual == "peptide" || qual == "anticodon" {
									val += strings.TrimSpace(txt)
								} else {
									val += " " + strings.TrimSpace(txt)
								}
							}

							rec.WriteString("          <INSDQualifier>\n")

							writeOneElement("            ", "INSDQualifier_name", qual)

							val = strings.TrimPrefix(val, "\"")
							val = strings.TrimSuffix(val, "\"")
							val = strings.TrimSpace(val)
							if val != "" {

								writeOneElement("            ", "INSDQualifier_value", val)
							}

							rec.WriteString("          </INSDQualifier>\n")
						}
					}
					if hasQual {
						rec.WriteString("        </INSDFeature_quals>\n")
					}

					// end of this feature
					rec.WriteString("      </INSDFeature>\n")
					// continue to next feature
				}
			}
			rec.WriteString("    </INSDSeq_feature-table>\n")

			// TSA, TLS, WGS, or CONTIG lines may be next

			alt_name := ""

			if strings.HasPrefix(line, "TSA") ||
				strings.HasPrefix(line, "TLS") ||
				strings.HasPrefix(line, "WGS") {

				alt.Reset()

				alt_name = line[:3]
				line = line[3:]
			}

			if strings.HasPrefix(line, "WGS_CONTIG") ||
				strings.HasPrefix(line, "WGS_SCAFLD") {

				alt.Reset()

				alt_name = line[:3]
				line = line[10:]
			}

			if alt_name != "" {

				alt_name = strings.ToLower(alt_name)
				txt := strings.TrimSpace(line)
				alt.WriteString(txt)
				for {
					// read next line
					line = nextLine()
					row++
					if !strings.HasPrefix(line, twelvespaces) {
						// if not continuation of contig, break out of loop
						break
					}
					// append subsequent line and continue with loop
					txt = strings.TrimPrefix(line, twelvespaces)
					txt = strings.TrimSpace(txt)
					alt.WriteString(txt)
				}
			}

			if strings.HasPrefix(line, "CONTIG") {

				// pathological records can have over 90,000 components, use strings.Builder
				con.Reset()

				txt := strings.TrimPrefix(line, "CONTIG")
				txt = strings.TrimSpace(txt)
				con.WriteString(txt)
				for {
					// read next line
					line = nextLine()
					row++
					if !strings.HasPrefix(line, twelvespaces) {
						// if not continuation of contig, break out of loop
						break
					}
					// append subsequent line and continue with loop
					txt = strings.TrimPrefix(line, twelvespaces)
					txt = strings.TrimSpace(txt)
					con.WriteString(txt)
				}
			}

			if strings.HasPrefix(line, "BASE COUNT") {

				txt := strings.TrimPrefix(line, "BASE COUNT")
				readContinuationLines(txt)
				// not supported
			}

			if strings.HasPrefix(line, "ORIGIN") {

				line = nextLine()
				row++
			}

			// remainder should be sequence

			// sequence can be millions of bases, use strings.Builder
			seq.Reset()

			for line != "" {

				if strings.HasPrefix(line, "//") {

					// end of record, print collected sequence
					str := seq.String()
					if str != "" {

						writeOneElement("    ", "INSDSeq_sequence", str)
					}
					seq.Reset()

					// print contig section
					str = con.String()
					str = strings.TrimSpace(str)
					if str != "" {
						writeOneElement("    ", "INSDSeq_contig", str)
					}
					con.Reset()

					if alt_name != "" {
						rec.WriteString("    <INSDSeq_alt-seq>\n")
						rec.WriteString("      <INSDAltSeqData>\n")
						str = alt.String()
						str = strings.TrimSpace(str)
						if str != "" {
							writeOneElement("        ", "INSDAltSeqData_name", alt_name)
							rec.WriteString("        <INSDAltSeqData_items>\n")
							writeOneElement("          ", "INSDAltSeqItem_value", str)
							rec.WriteString("        </INSDAltSeqData_items>\n")
						}
						alt.Reset()
						rec.WriteString("      </INSDAltSeqData>\n")
						rec.WriteString("    </INSDSeq_alt-seq>\n")
					}

					// end of record
					rec.WriteString("  </INSDSeq>\n")

					// send formatted record down channel
					txt := rec.String()
					out <- txt
					rec.Reset()
					// go to top of loop for next record
					break
				}

				// read next sequence line

				cols := strings.Fields(line)
				for _, str := range cols {

					if IsAllDigits(str) {
						continue
					}

					// append letters to sequence
					seq.WriteString(str)
				}

				// read next line and continue
				line = nextLine()
				row++

			}

			// continue to next record
		}
	}

	// launch single converter goroutine
	go convertGenBank(inp, out)

	return out
}

// STRING CONVERTERS

func EncodeURL(inp io.Reader) {

	if inp == nil {
		return
	}

	scanr := bufio.NewScanner(inp)

	for scanr.Scan() {

		str := scanr.Text()

		str = url.QueryEscape(str)

		os.Stdout.WriteString(str)
		if !strings.HasSuffix(str, "\n") {
			os.Stdout.WriteString("\n")
		}
	}
}

func DecodeURL(inp io.Reader) {

	if inp == nil {
		return
	}

	scanr := bufio.NewScanner(inp)

	for scanr.Scan() {

		str := scanr.Text()

		str, _ = url.QueryUnescape(str)

		os.Stdout.WriteString(str)
		if !strings.HasSuffix(str, "\n") {
			os.Stdout.WriteString("\n")
		}
	}
}

func EncodeB64(inp io.Reader) {

	if inp == nil {
		return
	}

	scanr := bufio.NewScanner(inp)

	for scanr.Scan() {

		str := scanr.Text()

		data := []byte(str)
		str = base64.StdEncoding.EncodeToString(data)

		os.Stdout.WriteString(str)
		if !strings.HasSuffix(str, "\n") {
			os.Stdout.WriteString("\n")
		}
	}
}

func DecodeB64(inp io.Reader) {

	if inp == nil {
		return
	}

	scanr := bufio.NewScanner(inp)

	for scanr.Scan() {

		str := scanr.Text()

		data, _ := base64.StdEncoding.DecodeString(str)
		str = string(data)

		os.Stdout.WriteString(str)
		if !strings.HasSuffix(str, "\n") {
			os.Stdout.WriteString("\n")
		}
	}
}

// MAIN FUNCTION

func main() {

	// skip past executable name
	args := os.Args[1:]

	if len(args) < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: No command-line arguments supplied to transmute\n")
		os.Exit(1)
	}

	// CONCURRENCY, CLEANUP, AND DEBUGGING FLAGS

	// do these first because -defcpu and -maxcpu can be sent from wrapper before other arguments

	ncpu := runtime.NumCPU()
	if ncpu < 1 {
		ncpu = 1
	}

	// wrapper can limit maximum number of processors to use (undocumented)
	maxProcs := ncpu
	defProcs := 0

	// concurrent performance tuning parameters, can be overridden by -proc and -cons
	numProcs := 0
	serverRatio := 4

	// number of servers usually calculated by -cons server ratio, but can be overridden by -serv
	NumServe = 0

	// number of channels usually equals number of servers, but can be overridden by -chan
	ChanDepth = 0

	// miscellaneous tuning parameters
	HeapSize = 16
	FarmSize = 64

	// garbage collector control can be set by environment variable or default value with -gogc 0
	goGc := 600

	// -flag sets -strict or -mixed cleanup flags from argument
	flgs := ""

	DeStop = true

	unicodePolicy := ""
	scriptPolicy := ""
	mathmlPolicy := ""

	// read data from file instead of stdin
	fileName := ""

	// debugging
	stts := false
	timr := false

	// profiling
	prfl := false

	// get numeric value
	getNumericArg := func(name string, zer, min, max int) int {

		if len(args) < 2 {
			fmt.Fprintf(os.Stderr, "\nERROR: %s is missing\n", name)
			os.Exit(1)
		}
		value, err := strconv.Atoi(args[1])
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: %s (%s) is not an integer\n", name, args[1])
			os.Exit(1)
		}
		// skip past first of two arguments
		args = args[1:]

		// special case for argument value of 0
		if value < 1 {
			return zer
		}
		// limit value to between specified minimum and maximum
		if value < min {
			return min
		}
		if value > max {
			return max
		}
		return value
	}

	inSwitch := true

	// get concurrency, cleanup, and debugging flags in any order
	for {

		inSwitch = true

		switch args[0] {
		// concurrency override arguments can be passed in by local wrapper script (undocumented)
		case "-maxcpu":
			maxProcs = getNumericArg("Maximum number of processors", 1, 1, ncpu)
		case "-defcpu":
			defProcs = getNumericArg("Default number of processors", ncpu, 1, ncpu)
		// performance tuning flags
		case "-proc":
			numProcs = getNumericArg("Number of processors", ncpu, 1, ncpu)
		case "-cons":
			serverRatio = getNumericArg("Parser to processor ratio", 4, 1, 32)
		case "-serv":
			NumServe = getNumericArg("Concurrent parser count", 0, 1, 128)
		case "-chan":
			ChanDepth = getNumericArg("Communication channel depth", 0, ncpu, 128)
		case "-heap":
			HeapSize = getNumericArg("Unshuffler heap size", 8, 8, 64)
		case "-farm":
			FarmSize = getNumericArg("Node buffer length", 4, 4, 2048)
		case "-gogc":
			goGc = getNumericArg("Garbage collection percentage", 0, 50, 1000)

		// read data from file
		case "-input":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Input file name is missing\n")
				os.Exit(1)
			}
			fileName = args[1]
			// skip past first of two arguments
			args = args[1:]

		// data cleanup flags
		case "-compress", "-compressed":
			DoCompress = true
		case "-spaces", "-cleanup":
			DoCleanup = true
		case "-strict":
			DoStrict = true
			AllowEmbed = true
			ContentMods = true
		case "-mixed":
			DoMixed = true
			AllowEmbed = true
			ContentMods = true
		case "-accent":
			DeAccent = true
		case "-ascii":
			DoASCII = true

		// previously visible processing flags (undocumented)
		case "-stems", "-stem":
			DoStem = true
		case "-stops", "-stop":
			DeStop = false

		// allow setting of unicode, script, and mathml flags (undocumented)
		case "-unicode":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Unicode argument is missing\n")
				os.Exit(1)
			}
			unicodePolicy = args[1]
			// skip past first of two arguments
			args = args[1:]
		case "-script":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Script argument is missing\n")
				os.Exit(1)
			}
			scriptPolicy = args[1]
			// skip past first of two arguments
			args = args[1:]
		case "-mathml":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: MathML argument is missing\n")
				os.Exit(1)
			}
			mathmlPolicy = args[1]
			// skip past first of two arguments
			args = args[1:]

		case "-flag", "-flags":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Flags argument is missing\n")
				os.Exit(1)
			}
			flgs = args[1]
			// skip past first of two arguments
			args = args[1:]

		// debugging flags
		case "-stats", "-stat":
			stts = true
		case "-timer":
			timr = true
		case "-profile":
			prfl = true

		default:
			// if not any of the controls, set flag to break out of for loop
			inSwitch = false
		}

		if !inSwitch {
			break
		}

		// skip past argument
		args = args[1:]

		if len(args) < 1 {
			break
		}
	}

	// -flag allows script to set -strict or -mixed (or -stems, or -stops) from argument
	switch flgs {
	case "strict":
		DoStrict = true
	case "mixed":
		DoMixed = true
	case "stems", "stem":
		DoStem = true
	case "stops", "stop":
		DeStop = false
	case "none", "default":
	default:
		if flgs != "" {
			fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized -flag value '%s'\n", flgs)
			os.Exit(1)
		}
	}

	UnicodeFix = ParseMarkup(unicodePolicy, "-unicode")
	ScriptFix = ParseMarkup(scriptPolicy, "-script")
	MathMLFix = ParseMarkup(mathmlPolicy, "-mathml")

	if UnicodeFix != NOMARKUP {
		DoUnicode = true
	}

	if ScriptFix != NOMARKUP {
		DoScript = true
	}

	if MathMLFix != NOMARKUP {
		DoMathML = true
	}

	// set dependent flags
	CountLines = DoMixed
	AllowEmbed = DoStrict || DoMixed
	ContentMods = AllowEmbed || DoCompress || DoUnicode || DoScript || DoMathML || DeAccent || DoASCII

	// reality checks on number of processors to use
	// performance degrades if capacity is above maximum number of partitions per second (context switching?)
	if numProcs == 0 {
		if defProcs > 0 {
			numProcs = defProcs
		} else {
			// best performance measurement with current code is obtained when 6 to 8 processors are assigned,
			// varying slightly among queries on PubmedArticle, gene DocumentSummary, and INSDSeq sequence records
			numProcs = 8
			if cpuid.CPU.ThreadsPerCore > 1 {
				cores := ncpu / cpuid.CPU.ThreadsPerCore
				if cores > 4 && cores < 8 {
					numProcs = cores
				}
			}
		}
	}
	if numProcs > ncpu {
		numProcs = ncpu
	}
	if numProcs > maxProcs {
		numProcs = maxProcs
	}

	// allow simultaneous threads for multiplexed goroutines
	runtime.GOMAXPROCS(numProcs)

	// adjust garbage collection target percentage
	if goGc >= 50 && goGc <= 1000 {
		debug.SetGCPercent(goGc)
	}

	// explicit -serv argument overrides -cons ratio
	if NumServe > 0 {
		serverRatio = NumServe / numProcs
		// if numServers / numProcs is not a whole number, do not print serverRatio in -stats
		if NumServe != numProcs*serverRatio {
			serverRatio = 0
		}
	} else {
		NumServe = numProcs * serverRatio
	}
	// server limits
	if NumServe > 128 {
		NumServe = 128
	} else if NumServe < 1 {
		NumServe = numProcs
	}

	// explicit -chan argument overrides default to number of servers
	if ChanDepth == 0 {
		ChanDepth = NumServe
	}

	// -stats prints number of CPUs and performance tuning values if no other arguments (undocumented)
	if stts && len(args) < 1 {

		fmt.Fprintf(os.Stderr, "Core %d\n", ncpu/cpuid.CPU.ThreadsPerCore)
		fmt.Fprintf(os.Stderr, "Thrd %d\n", ncpu)
		fmt.Fprintf(os.Stderr, "Sock %d\n", ncpu/cpuid.CPU.LogicalCores)
		fmt.Fprintf(os.Stderr, "Mmry %d\n", memory.TotalMemory()/(1024*1024*1024))

		fmt.Fprintf(os.Stderr, "Proc %d\n", numProcs)
		if serverRatio > 0 {
			fmt.Fprintf(os.Stderr, "Cons %d\n", serverRatio)
		}
		fmt.Fprintf(os.Stderr, "Serv %d\n", NumServe)
		fmt.Fprintf(os.Stderr, "Chan %d\n", ChanDepth)
		fmt.Fprintf(os.Stderr, "Heap %d\n", HeapSize)
		fmt.Fprintf(os.Stderr, "Farm %d\n", FarmSize)
		fmt.Fprintf(os.Stderr, "Gogc %d\n", goGc)

		fi, err := os.Stdin.Stat()
		if err == nil {
			mode := fi.Mode().String()
			fmt.Fprintf(os.Stderr, "Mode %s\n", mode)
		}

		fmt.Fprintf(os.Stderr, "\n")

		return
	}

	if len(args) < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to transmute\n")
		os.Exit(1)
	}

	// DOCUMENTATION COMMANDS

	inSwitch = true

	switch args[0] {
	case "-version":
		fmt.Printf("%s\n", transmuteVersion)
	case "-help":
		fmt.Printf("transmute %s\n%s\n", transmuteVersion, transmuteHelp)
	default:
		// if not any of the documentation commands, keep going
		inSwitch = false
	}

	if inSwitch {
		return
	}

	// FILE NAME CAN BE SUPPLIED WITH -input COMMAND

	in := os.Stdin

	// check for data being piped into stdin
	isPipe := false
	fi, err := os.Stdin.Stat()
	if err == nil {
		isPipe = bool((fi.Mode() & os.ModeNamedPipe) != 0)
	}

	usingFile := false

	if fileName != "" {

		inFile, err := os.Open(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName)
			os.Exit(1)
		}

		defer inFile.Close()

		// use indicated file instead of stdin
		in = inFile
		usingFile = true

		if isPipe && runtime.GOOS != "windows" {
			mode := fi.Mode().String()
			fmt.Fprintf(os.Stderr, "\nERROR: Input data from both stdin and file '%s', mode is '%s'\n", fileName, mode)
			os.Exit(1)
		}
	}

	// check for -input command after extraction arguments
	for _, str := range args {
		if str == "-input" {
			fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -input command\n")
			os.Exit(1)
		}
	}

	// START PROFILING IF REQUESTED

	if prfl {

		f, err := os.Create("cpu.pprof")
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create profile output file\n")
			os.Exit(1)
		}

		pprof.StartCPUProfile(f)

		defer pprof.StopCPUProfile()
	}

	// INITIALIZE RECORD COUNT

	recordCount := 0
	byteCount := 0

	// print processing rate and program duration
	printDuration := func(name string) {

		PrintDuration(name, recordCount, byteCount)
	}

	// JSON TO XML CONVERTER

	if args[0] == "-j2x" || args[0] == "-json2xml" {

		set := "root"
		rec := ""
		nest := ""

		nextArg := func() (string, bool) {

			if len(args) < 1 {
				return "", false
			}

			// remove next token from slice
			nxt := args[0]
			args = args[1:]

			return nxt, true
		}

		// look for optional arguments
		args = args[1:]
		for {
			arg, ok := nextArg()
			if !ok {
				break
			}

			switch arg {
			case "-set":
				// override set wrapper
				set, ok = nextArg()
				if ok && set == "-" {
					set = ""
				}
			case "-rec":
				// override record wrapper
				rec, ok = nextArg()
				if ok && rec == "-" {
					rec = ""
				}
			case "-nest":
				// specify nested array naming policy
				nest, ok = nextArg()
				if !ok {
					fmt.Fprintf(os.Stderr, "Nested array naming policy is missing\n")
					os.Exit(1)
				}
				if ok && nest == "-" {
					nest = "flat"
				}
				switch nest {
				case "flat", "plural", "name", "recurse", "recursive", "same", "depth", "deep", "level":
				default:
					fmt.Fprintf(os.Stderr, "Unrecognized nested array naming policy\n")
					os.Exit(1)
				}
			default:
				// alternative form uses positional arguments to override set and rec
				set = arg
				if set == "-" {
					set = ""
				}
				rec, ok = nextArg()
				if ok && rec == "-" {
					rec = ""
				}
			}
		}

		// use output channel of tokenizer as input channel of converter
		jtkn := JSONTokenizer(in)
		jcnv := JSONConverter(jtkn, set, rec, nest)

		if jtkn == nil || jcnv == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create JSON to XML converter\n")
			os.Exit(1)
		}

		if set != "" {
			fmt.Fprintf(os.Stdout, "<%s>\n", set)
		}

		// drain output of last channel in service chain
		for str := range jcnv {

			if str == "" {
				continue
			}

			// send result to output
			os.Stdout.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				os.Stdout.WriteString("\n")
			}

			recordCount++
			runtime.Gosched()
		}

		if set != "" {
			fmt.Fprintf(os.Stdout, "</%s>\n", set)
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("blocks")
		}

		return
	}

	// ASN.1 TO XML CONVERTER

	if args[0] == "-a2x" || args[0] == "-asn2xml" {

		set := ""
		rec := ""

		nextArg := func() (string, bool) {

			if len(args) < 1 {
				return "", false
			}

			// remove next token from slice
			nxt := args[0]
			args = args[1:]

			return nxt, true
		}

		// look for optional arguments
		args = args[1:]
		for {
			arg, ok := nextArg()
			if !ok {
				break
			}

			switch arg {
			case "-set":
				// override set wrapper
				set, ok = nextArg()
				if ok && set == "-" {
					set = ""
				}
			case "-rec":
				// override record wrapper
				rec, ok = nextArg()
				if ok && rec == "-" {
					rec = ""
				}
			}
		}

		atkn := ASN1Tokenizer(in)
		acnv := atkn
		acnv = ASN1Converter(atkn, set, rec)

		if atkn == nil || acnv == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create ASN.1 to XML converter\n")
			os.Exit(1)
		}

		// drain output of last channel in service chain
		for str := range acnv {

			if str == "" {
				continue
			}

			// send result to output
			os.Stdout.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				os.Stdout.WriteString("\n")
			}

			recordCount++
			runtime.Gosched()
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("blocks")
		}

		return
	}

	// READ TAB-DELIMITED FILE AND WRAP IN XML FIELDS

	// must be called before CreateReader starts draining stdin
	if len(args) > 1 && args[0] == "-t2x" {

		recordCount = TableConverter(in, "\t", args)

		debug.FreeOSMemory()

		if timr {
			printDuration("lines")
		}

		return
	}

	if len(args) > 1 && args[0] == "-c2x" {

		recordCount = TableConverter(in, ",", args)

		debug.FreeOSMemory()

		if timr {
			printDuration("lines")
		}

		return
	}

	// READ GENBANK FLATFILE AND TRANSLATE TO INSDSEQ XML

	// must be called before CreateReader starts draining stdin
	if len(args) > 0 && args[0] == "-g2x" {

		gbk := GenBankConverter(in)

		if gbk == nil {
			fmt.Fprintf(os.Stderr, "Unable to create GenBank to XML converter\n")
			os.Exit(1)
		}

		head := `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE INSDSet PUBLIC "-//NCBI//INSD INSDSeq/EN" "http://www.ncbi.nlm.nih.gov/dtd/INSD_INSDSeq.dtd">
<INSDSet>
`
		tail := ""

		// drain output of last channel in service chain
		for str := range gbk {

			if str == "" {
				continue
			}

			if head != "" {
				os.Stdout.WriteString(head)
				head = ""
				tail = `</INSDSet>
`
			}

			// send result to stdout
			os.Stdout.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				os.Stdout.WriteString("\n")
			}

			recordCount++

			runtime.Gosched()
		}

		if tail != "" {
			os.Stdout.WriteString(tail)
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	// STRING CONVERSION COMMANDS

	inSwitch = true

	switch args[0] {
	case "-encodeURL":
		EncodeURL(in)
	case "-decodeURL":
		DecodeURL(in)
	case "-encodeB64":
		EncodeB64(in)
	case "-decodeB64":
		DecodeB64(in)
	default:
		// if not any of the conversion commands, keep going
		inSwitch = false
	}

	if inSwitch {

		debug.FreeOSMemory()

		return
	}

	// CREATE XML BLOCK READER FROM STDIN OR FILE

	rdr := CreateReader(in)
	if rdr == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n")
		os.Exit(1)
	}

	// CONFIRM INPUT DATA AVAILABILITY AFTER RUNNING COMMAND GENERATORS

	if fileName == "" && runtime.GOOS != "windows" {

		fromStdin := bool((fi.Mode() & os.ModeCharDevice) == 0)
		if !isPipe || !fromStdin {
			mode := fi.Mode().String()
			fmt.Fprintf(os.Stderr, "\nERROR: No data supplied to transmute from stdin or file, mode is '%s'\n", mode)
			os.Exit(1)
		}
	}

	if !usingFile && !isPipe {

		fmt.Fprintf(os.Stderr, "\nERROR: No XML input data supplied to transmute\n")
		os.Exit(1)
	}

	// SPECIAL FORMATTING COMMANDS

	inSwitch = true
	leaf := false

	switch args[0] {
	case "-format":
		recordCount = ProcessFormat(rdr, args, timr)
	case "-filter":
		ProcessFilter(rdr, args)
	case "-normalize", "-normal":
		DoMixed = true
		AllowEmbed = true
		ContentMods = true
		ProcessNormalize(rdr, args)
	case "-outline":
		ProcessOutline(rdr)
	case "-contour":
		leaf = true
		fallthrough
	case "-synopsis":
		args = args[1:]
		delim := "/"
		if len(args) > 0 {
			delim = args[0]
			if len(delim) > 3 {
				delim = "/"
			}
		}
		ProcessSynopsis(rdr, leaf, delim)
	case "-tokens":
		ProcessTokens(rdr)
	default:
		// if not any of the formatting commands, keep going
		inSwitch = false
	}

	if inSwitch {

		debug.FreeOSMemory()

		// suppress printing of lines if not properly counted
		if recordCount == 1 {
			recordCount = 0
		}

		if timr {
			printDuration("lines")
		}

		return
	}

	// force garbage collection and return memory before calculating processing rate
	debug.FreeOSMemory()

	if timr {
		printDuration("records")
	}

	// REPORT UNRECOGNIZED COMMAND

	fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized transmute command\n")
	os.Exit(1)
}
