From 7997171fa137a6cc64ea8f63584d248403d316fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Mon, 21 Jun 2021 08:05:35 -0700 Subject: [PATCH 1/3] Add story and doc folders and support Stories are short essays explaining more involved YAML usage and ideas. Docs are developer documentation Added build system support for building them into the site. Also added `make stage` which publishes to: http://-spec.yaml.io for showing live content for pull requests. --- Makefile | 2 +- doc/ReadMe.md | 6 + doc/glossary.md | 444 ++++++++++++++++++ doc/index.md | 7 + story/ReadMe.md | 10 + story/implicit-merging.md | 109 +++++ story/index.md | 11 + tool/bin/get-fork-id | 7 + .../{render-spec-markdown => render-markdown} | 0 ...er-spec-markdown.pl => render-markdown.pl} | 63 ++- www/.gitignore | 2 +- www/Makefile | 143 ++++-- www/ReadMe.md | 119 ++++- www/index.md | 16 +- www/{spec-plain.md.yaml => spec-plain.md} | 1 + www/spec.md.yaml | 3 - 16 files changed, 870 insertions(+), 73 deletions(-) create mode 100644 doc/ReadMe.md create mode 100644 doc/glossary.md create mode 100644 doc/index.md create mode 100644 story/ReadMe.md create mode 100644 story/implicit-merging.md create mode 100644 story/index.md create mode 100755 tool/bin/get-fork-id rename tool/bin/{render-spec-markdown => render-markdown} (100%) rename tool/lib/{render-spec-markdown.pl => render-markdown.pl} (87%) rename www/{spec-plain.md.yaml => spec-plain.md} (63%) delete mode 100644 www/spec.md.yaml diff --git a/Makefile b/Makefile index f6e5367f..96a2b8f4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ default: docker-build-all docker-push-all: make -C tool/docker $@ -build serve publish: +build serve stage publish: make -C www $@ clean: diff --git a/doc/ReadMe.md b/doc/ReadMe.md new file mode 100644 index 00000000..16b93e31 --- /dev/null +++ b/doc/ReadMe.md @@ -0,0 +1,6 @@ +YAML Developer Documentation +============================ + +This directory contains documents and manuals that are critical for YAML +framework developers to read and understand, in order to build correct and +robust YAML processors. diff --git a/doc/glossary.md b/doc/glossary.md new file mode 100644 index 00000000..10425985 --- /dev/null +++ b/doc/glossary.md @@ -0,0 +1,444 @@ +YAML Vocabulary Glossary +======================== + +The YAML language has its own vocabulary. +It is important to use the correct terms when developers discuss YAML topics. +For example the terms: `list`, `array` and `sequence` seem like they are the +same thing and can be used interchangably, but in YAML they have different +meanings. + +## A + +* Alias + + An alias is a reference to another YAML node. + It similar to a pointer in the C programming language. + The alias `*foo` is a reference to a node with an anchor `&foo`. + +* Anchor + + An anchor is a label attached to a node. + The node can be used elsewhere by referencing it with an alias. + A node with the anchor `&foo` can be referenced elsewhere with the alias + `*foo`. + +* Application + + An end user program using a YAML framework. + +* Array + + The word array is sometimes used to refer to a native data structure + typically constructed from a sequence node. + +## B + +* Block / Block Collection Style + + Block collections are written in the indentation-based scoping style that + YAML is most known for. + +* Boolean + + Boolean is a native state consisting of the values true and false. + Most constructors/schemas support booleans. + It is typical for a schema to use the plain scalars `true` and `false` to + assign the boolean tag function. + +## C + +* Comment + + Text in a YAML file that is typically intended for humans to read, but not + considered part of the YAML data model. + Comment text may be discarded entirely by a parser. + It may also be reported as events and stored in the DOM or possibly in a + native data structure. + Syntactically, comments are text starting with a `#` character and continuing + to the end of the line. + + Comments may be used within YAML documents or before/between/after them (in + the YAML stream). + +* Composer + + A composer is the processor in a load stack that gets events from a parser + and uses them to create the DOM state. + +* Constructor + + A constructor is the processor in a load stack that iterates over a DOM tree + and creates a native representation. + +## D + +* Directive + + A directive is an instruction to the parser. + YAML 1.2 defines 2 directives: `%YAML ...` and `%TAG ...`. + Directives are part of a YAML stream, but not part of YAML documents or the + YAML data model. + +* Document + + A document is a top level YAML node. + Most YAML files consist of a single YAML document, although they may also + have zero or multiple documents. + +* Double quoted scalar + + Scalar values written in the double quote style are capable of expressing any + possible string value. + The double quoted style is the only scalar style capable of that. + They use a number of escape sequences to represent non-printable characters. + +* DOM + + A DOM is an information state that is a tree of created by a composer or a + representer and consumed by a constructor or a serializer. + A DOM may also be consumed directly by an application. + A DOM API can offer a lot more information and processing options than a + native data structure. + + Frameworks do not need to implement a DOM as part of their stacks, as long as + they adhere to the YAML specification rules for moving information through + the DOM. + +* Dumper / Dump + + A dumper is a processor that links all the processors of a dump stack, taking + information all the way from native to file states. + +* Dump Stack + + The set of processors that move YAML information from a native state to a + file state. + +## E + +* Emitter / Emit + + An emitter is the process in the dump stack that turns events into tokens. + In the dump stack the events come from the serializer. + In stream processing the events would come from an application filter process + that would typically be reading events from a parser. + +* Event + + An event is an information state produced by a parser or serializer and + consumed by a composer or emitter. + Event types include: + + * stream-start + * stream-end + * document-start + * document-end + * mapping-start + * mapping-end + * sequence-start + * sequence-end + * scalar-value + * alias-name + +## F + +* File + + File is the term for YAML information in a final textual state, external to + the yaml stack. + A YAML framework loads from a file and dumps to a file. + The term is abstract and doesn't have to be a file stored to disk. + It might be a socket or other external data source/target. + +* Flow / Flow Collection Style + + In YAML, mappings and sequences can be represented in a style that uses curly + braces and square brackets in the same manner that JSON does. + Block collections may contain any collection nodes in the flow style, but + flow collections may only contain collections in the flow style. + +* Folded Scalar Style + + Folded scalars occur only in block collections. + They are indicated by a greater-than sign. + The indented lines that follow replace newlines with a space, and two or more + consecutive newlines with n-1 literal newlines. + +* Framework / YAML Framework + + A full YAML processing implementation in a given programming language. + A framework almost always has at least a Loader and a Dumper. + A complete, full-featured YAML framework would also support things like + schema processors, path referencing, DOM API and standard library support, + among many other details. + +## H + +* Hash + + The word hash is sometimes used to refer to a native data structure typically + constructed from a mapping node. + +## I + +* Information + + Any of the various data bits flowing through a YAML stack. + +## J + +* JSON + + When used in with a the JSON Schema or a derivative of that schema, YAML is a + syntactic and semantic superset of the JSON data format. + That is, a YAML loader using such a schema (which is typical) can load a JSON + file and produce the same result as a JSON loading (often called `parse`) + process. + +## K + +* Kind + + There are 3 kinds of nodes in YAML: mappings, sequences and scalars. + A kind refers to the raw structure. + A kind should not be confused with a type. + +## L + +* Library + + The DOM resolves tags to functions. + These functions come from the library that is registered to the DOM; often + the yaml standard library. + +* List + + A list is a property of a type. + List types are often made from sequence nodes, but they can be result of any + tag function whose return type is a list type. + +* Literal / Literal Scalar Style + + The literal scalar is similar to the heredoc style found in some programming + languages like Perl and Bash. + No character escaping is allowed and newlines are literal. + Any valid YAML file's content can be encoded in the literal style by simply + indenting it. + +* Loader / Load + + A dumper is a processor that links all the processors of a dump stack, taking + information all the way from native to file states. + +* Load Stack + + The set of processors that move YAML information from the file state to the + native state: read, lex, parse, compose, construct. + +## M + +* Mapping + + A mapping is a kind of YAML node that consists of a set of zero or more + key/value pairs. + Any kind of node is allowed to be a key, even though native models rarely + support this. + Equivalent keys (if they can be detected) are not supported. + +* Model / Data Model + + This word describes elementary kinds of data that YAML represents. + A YAML file is a stream of YAML documents each of which have one root node. + Nodes may be mappings, sequences or scalars. + Nodes may be annotated with anchors and/or tags. + An alias may be used for any node. + +## N + +* Native / Native Object + + A language specific state that is the final result of a loader, or the + initial state given to a dumper. + This state is defined by the programming language being used, or maybe a form + crafted by the author of the application. + Some YAML frameworks may use the DOM state as their native state. + + As an example, in Python, generic native states include dictionaries, lists, + tuples and values. + Custom native states include the instance objects of Python classes. + +* Node + + A node is an addressable point in a YAML document. + Mappings, sequences, scalars are nodes. + An alias is a reference to a node. + Anchors and tags are annotations to nodes. + +* Null + + Null is a native value supported by most schemas. + The plain value `null` as well as the plain empty value is most often used to + represent it. + +## P + +* Parser / Parse + + A parser is the processor in the load stack that reads tokens, matches them + against a grammar and write events. + It make also throw an error if the tokens don't match the grammar. + The events are usually consumed by the composer to create a DOM, but they + might also be processed directly by a streaming application. + +* Plain Scalar + + Plain refers to the quoting style of a scalar where the value is unquoted; as + opposed to single/double quoted or literal or folded styles. + A scalar-value event contains a flag as to whether the scalar was plain or + not. + Plain scalars are often assigned tags based on their content value. + +* Processor + + A component in the load stack or dump stack to transforms data from one state + to another. + Load stack processors include: reader, lexer, parser, composer and + constructor. + Dump stack processors include: representer, serializer, emitter, streamer and + writer. + +## R + +* Reader / Read + + A reader is a processor in the load stack that reads a file and produces a + stream of unicode characters. + +* Reference + + A reference can be thought of as a pointer to another node in the DOM. + In YAML 1.2 the only references are aliases. + +* Representer / Represent + + A representer is a process that turns native programming data into a YAML DOM + state. + +## S + +* Scalar + + A scalar is a a leaf node that contains exactly one value. + Strings, numbers and booean values are examples of scalars. + +* Schema + + Schema in YAML refers to all the external information required to process a + YAML information. + Unlike traditional schemas which typically enforce the structural typing of + information, a YAML schema can alter the semantic meaning of a YAML file. + + In YAML 1.2, schemas are almost always expressed in the source code of the + framework. + In future versions, a YAML Schema language can be used to control the + behavior of a framework (if the framework supports it). + +* Sequence + + A sequence is a collection node that consists of zero or more nodes. + +* Serializer / Serialize + + A serializer is the process in the dump stack that iterates over the DOM and + produces events that are typically sent to an emitter. + +* Single quoted scalar + + Scalar values written in the single quote style can contain any sequence of + printable characters except the single quote itself. + The single quotes must be escaped using two single quotes (`''`). + +* Stack / YAML Stack + + YAML processing occurs as 2 stacks of processors, each moving data (in + opposite directions) between YAML formatted text and computer memory states. + They are called the load stack and the dump stack. + Collectively the 2 stacks may be referred to as "the YAML stack". + +* Standard Library + + Versions of YAML after 1.2 define a standard library of tag functions. + +* State + + A form that YAML information is in during various stages of the stack. + States include: files, characters, tokens, events, DOMs and native objects. + For instance an "event" is a data state produced by a parser or a serializer, + and consumed by a composer or an emitter. + +* Stream (Character Stream) + + The set of unicode characters produced by a reader or streamer and consumed + by a lexer or writer. + +* Stream (YAML Stream or Document Stream) + + A YAML file is considered a "stream" of zero or more documents. + A stream may also have directives and/or comments before, between or after + the documents. + This is the typical meaning when the word "stream" is used with no qualifier. + +* Streamer + + This is the dump stack sister process to the lexer. + It simply joins tokens into a character stream. + +* String + + String is a heavily overloaded term in YAML and depends on the context in + which it is used. + +* Style + + YAML has multiple styles to represent collections and scalars. + Collection styles are "block" and "flow". + Scalar styles are "plain", "single quoted", "double quoted", "literal" and + "folded". + +## T + +* Tag + + A tag is an annotation on a node. + Tags are identifers preceded by a `!`, like `!foo` or `!!str`. + A tag identifier is used to identify a function that will be applied to a + node when it is retrieved from the DOM. + The function's return type is can be considered the "type" of the node. + +* Token + + A token is a piece of information that is produced by a lexer or emitter. + A token has a name and a value. + The value is a sequence of zero or more contiguous characters from (or for) a + character stream. + +* Type + + A type is a set of constraints on a node. + Types are defined by schemas. + +## W + +* Writer / Write + + A writer is the processor in a dump stack that encodes unicode characters and + writes them to a file state. + +## Y + +* YAML + + YAML is a programming-language-agnostic data serialization language. + "YAML" rhymes with "camel". + YAML is a recursive backronym that stands for "YAML Ain't Markup Language". + People often think YAML is Yet Another Markup Language, but it Ain't! diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 00000000..65562b8a --- /dev/null +++ b/doc/index.md @@ -0,0 +1,7 @@ +YAML Developer Documentation +============================ + +The following documentation pages are important for the devlopers of +YAML frameworks to understand. + +* [YAML Vocabulary Glossary](glossary) diff --git a/story/ReadMe.md b/story/ReadMe.md new file mode 100644 index 00000000..4ab29f37 --- /dev/null +++ b/story/ReadMe.md @@ -0,0 +1,10 @@ +story +===== + +The `story/` directory contains various Story documents. + +A story is a narrative about a specific compelling use case, and how it works +or would work in the future. + +Stories help the creation of RFCs and help people understand the less obvious +things you can do with YAML. diff --git a/story/implicit-merging.md b/story/implicit-merging.md new file mode 100644 index 00000000..cd792ab5 --- /dev/null +++ b/story/implicit-merging.md @@ -0,0 +1,109 @@ +Implicit Merging +================ + +A mapping merge looks like this: +``` +--- &base +foo: 1 +bar: 2 +--- !merge +aaa: +- *base +- baz: 3 + bar: 4 +- xxx: 9 +``` + +Resulting in: +``` +aaa: + foo: 1 + baz: 3 + bar: 4 + xxx: 9 +``` + +That is, a merge is takes a sequence of two or more mappings and merges them, +with latter pairs taking precedence. + +The YAML 1.2 "merge key" can be done with a schema like this: +``` ++map: + base: ++map + when: + - pkey: '<<' + func: merge-key +``` + +Effectively that makes this YAML: +``` +aaa: + <<: *base + baz: 3 + bar: 4 +``` + +Be loaded like: +``` +aaa: !merge-key + <<: *base + baz: 3 + bar: 4 +``` + +In other words, it implicitly tags a mapping. + +We can also implicitly tag a value that expects a mapping type, but has a +sequence of mappings. The schema looks like: +``` ++map: + base: ++map + when: + - list: +map + func: merge +``` + +That makes this YAML: +``` +aaa: +- *base +- baz: 3 + bar: 4 +- xxx: 9 +``` + +Be loaded like: +``` +aaa: !merge +- *base +- baz: 3 + bar: 4 +- xxx: 9 +``` + +This is another way to implicitly tag a mapping. + +Some 1.1 and 1.2 loaders do the `merge-key` style merging by default. +It would not make sense to do the latter `merge` tag by default, but it +certainly might make sense for a specific application's schema. +It could even be applied only to certain specific nodes in a document. + +All it does is save you from needing to explicitly tag a node that the schema +says must be a mapping, but the loader finds a sequence of mappings instead. + +This is an example of how a YAML loader can expose the full power of tag +functional transforms, without the YAML ever needing to have explicit tags +actually in the YAML. + +Just like: +``` +- 123 +``` +is more natural to write than: +``` +- !int '123' +``` + +It's the exact same thing for collections. +Of course, an author is always free to use the `!merge` tag explicitly (even +though the schema dictates it) if they feel the intent might be lost on others. diff --git a/story/index.md b/story/index.md new file mode 100644 index 00000000..ec6ce477 --- /dev/null +++ b/story/index.md @@ -0,0 +1,11 @@ +# YAML Stories + +YAML Stories are short essays on how YAML currently works, or might work in the +future. +Each story describes a particular situation and/or sets of data and how they +end up getting processed. + +## Links + +* [Implicit Merging of Mappings](implicit-merging) + diff --git a/tool/bin/get-fork-id b/tool/bin/get-fork-id new file mode 100755 index 00000000..3d63bc03 --- /dev/null +++ b/tool/bin/get-fork-id @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +remote=$(git config remote.fork.url) || exit 0 +[[ $remote == git@github.com:*/yaml-spec* ]] || exit 0 +remote=${remote#git@github.com:} +remote=${remote%/*} +echo $remote diff --git a/tool/bin/render-spec-markdown b/tool/bin/render-markdown similarity index 100% rename from tool/bin/render-spec-markdown rename to tool/bin/render-markdown diff --git a/tool/lib/render-spec-markdown.pl b/tool/lib/render-markdown.pl similarity index 87% rename from tool/lib/render-spec-markdown.pl rename to tool/lib/render-markdown.pl index b1f6c945..922ca1dc 100644 --- a/tool/lib/render-spec-markdown.pl +++ b/tool/lib/render-markdown.pl @@ -11,12 +11,12 @@ my $links = {}; sub main { - my ($front, $markdown, $links_hash) = read_files(@_); + my ($front, $markdown, $link_map) = read_files(@_); my $parsed = parse_sections($markdown); set_vars(); - make_link_index($parsed, $links_hash); + make_link_index($parsed, $link_map); my @sections; for my $section (@$parsed) { @@ -48,6 +48,9 @@ sub parse_sections { my @s; while ($_) { # Headings: + s/\A(\S.*\n)===+\n\n+// + and push @s, {heading => "# $1"} and next; + s/\A(#{1,5} .*\n)\n+// and push @s, {heading => $1} and next; @@ -64,7 +67,17 @@ sub parse_sections { s/\A ( (::.*\n)? - (?:\*\ .*\n)+ + (?: + (?:\*\ \S.*\n) + (?: + (?: + \n + (\ \ \S.*\n)+ + )+ + \n? + )? + (?:\ \ \*\ .*\n)* + )+ (?:\{:\.\S+\}\n)? ) \n+ @@ -78,7 +91,11 @@ sub parse_sections { .*\n )+ ) - \n+ + (?: + (?=```) | + \z | + \n+ + ) //x and push @s, {p => $1} and next; @@ -112,9 +129,14 @@ sub parse_sections { s/\A((?:\* .*\n)(?: .*\n|\n(?= ))+)\n+// and push @s, {ul => $1} and next; + s/\A(\n)\n*// + and push @s, {comment => $1} and next; + + s/\n+\z// and next; + s/((?:.*\n){20})(?s:.*)/$1/; WWW(\@s); - die "------\n$_<<<"; + die "*** ERROR ***\nParse failed at this point:\n$_\n*** EOF ***\n"; } return \@s; @@ -232,6 +254,8 @@ sub fmt_pre { $_ = $pre; } +sub fmt_comment {} + sub fmt_img {} sub fmt_table {} @@ -251,10 +275,31 @@ sub fmt_index { #------------------------------------------------------------------------------ sub read_files { - my ($front_file, $markdown_file, $links_file) = @_; - my $front = read_file($front_file); - my $markdown = read_file($markdown_file); - my $links = YAML::PP::LoadFile($links_file); + my ($root, $markdown_file, $links_file) = @_; + my ($front, $markdown, $links); + $markdown = read_file($markdown_file); + if ($markdown =~ s/\A(---\n.*?\n---\n)//s) { + $front = $1; + } + else { + $front = "---\n---\n"; + } + if ($links_file) { + $links = YAML::PP::LoadFile($links_file); + } + else { + $links = {}; + } + my $yaml = $front; + $yaml =~ s/\A---\n//; + $yaml =~ s/---\n\z//; + if ($yaml) { + my $data = YAML::PP::Load($yaml); + if (my $source = $data->{source}) { + $markdown = read_file("$root/$source"); + } + } + $markdown .= "\n"; return ($front, $markdown, $links) } diff --git a/www/.gitignore b/www/.gitignore index 953b974c..e9995eb7 100644 --- a/www/.gitignore +++ b/www/.gitignore @@ -1,2 +1,2 @@ -/_stage/ +/_site/ /_gh-pages/ diff --git a/www/Makefile b/www/Makefile index 55079ed5..f58949a8 100644 --- a/www/Makefile +++ b/www/Makefile @@ -5,14 +5,29 @@ PUBLISH_CNAME := spec.yaml.io SPEC12 := $(ROOT)/1.2 SPEC := $(ROOT)/spec +DOC := $(ROOT)/doc +STORY := $(ROOT)/story WORK := $(ROOT)/work -STAGE := _stage -PAGES := _gh-pages -LINKS := $(ROOT)/spec/links.yaml -IMAGE := $(shell ls $(SPEC)/img/) +SITE := _site +GH_PAGES := _gh-pages + +SPEC_LINKS := $(SPEC)/links.yaml +SPEC_IMAGE := $(shell ls $(SPEC)/img/) SPEC_MD_HTML := $(WORK)/markdown.html INPUT_HTML := $(WORK)/spec-1.2.html +DOC_MD := $(shell \ + find $(DOC) -name '*.md' | \ + grep -v ReadMe.md \ +) +DOC_MD := $(DOC_MD:$(ROOT)/%=%) + +STORY_MD := $(shell \ + find $(STORY) -name '*.md' | \ + grep -v ReadMe.md \ +) +STORY_MD := $(STORY_MD:$(ROOT)/%=%) + DOCKER_BUILD_OPTS := \ --user $(UID):$(GID) \ @@ -24,104 +39,140 @@ DOCKER_SHELL_OPTS := \ $(DOCKER_SERVE_OPTS) \ --volume $(HISTORY_FILE):/home/jekyll/.bash_history \ -CMD ?= bash +SHELL_CMD ?= bash -FILES := $(shell \ +WWW_FILES := $(shell \ find . -type f -name '*.md*' | \ grep -v ReadMe | \ grep -v /_ \ ) -FILES := $(FILES:./%=%) -FILES := $(FILES:%.yaml=%) - -FILES := \ - $(FILES) \ - $(IMAGE) \ +WWW_FILES := $(WWW_FILES:./%=%) +WWW_FILES := $(WWW_FILES:%.yaml=%) + +SITE_FILES := \ + $(WWW_FILES) \ + spec.md \ + $(SPEC_IMAGE) \ + $(DOC_MD) \ + $(STORY_MD) \ Gemfile \ _config.yml \ _layouts \ - index.md \ review.html \ - spec-plain.md \ spec.scss \ img \ -FILES := $(FILES:%=$(STAGE)/%) +SITE_FILES := $(SITE_FILES:%=$(SITE)/%) +SITE_FILES := $(SITE_FILES:%.swp=) JEKYLL_BUILD := jekyll build JEKYLL_SERVE := jekyll serve --host 0.0.0.0 +FORK_USER_ID := $(shell $(ROOT)/tool/bin/get-fork-id) -list-files: - @printf "%s\n" $(FILES) +site-files: + @printf "%s\n" $(SITE_FILES) | sort -stage: $(FILES) $(PAGES) +site: $(SITE) $(SITE_FILES) $(GH_PAGES) -build: stage - $(eval override export YAML_SPEC_DIR := www/$(STAGE)) +build: site + $(eval override export YAML_SPEC_DIR := www/$(SITE)) $(call docker-run,run $(JEKYLL_BUILD),$(DOCKER_BUILD_OPTS)) - echo $(PUBLISH_CNAME) > $(PAGES)/CNAME - cp $(PAGES)/spec.html $(WORK)/ + echo $(PUBLISH_CNAME) > $(GH_PAGES)/CNAME + cp $(GH_PAGES)/spec.html $(WORK)/ -serve: stage - $(eval override export YAML_SPEC_DIR := www/$(STAGE)) +serve: site + $(eval override export YAML_SPEC_DIR := www/$(SITE)) $(call docker-run,run $(JEKYLL_SERVE),$(DOCKER_SERVE_OPTS)) -shell: stage - $(eval override export YAML_SPEC_DIR := www/$(STAGE)) - $(call docker-run,run $(CMD),$(DOCKER_SHELL_OPTS)) +shell: site + $(eval override export YAML_SPEC_DIR := www/$(SITE)) + $(call docker-run,run $(SHELL_CMD),$(DOCKER_SHELL_OPTS)) + +stage: build +ifeq ($(FORK_USER_ID),) + @echo '*** ERROR' + @echo "No git remote called 'fork' found" + @echo "It should have a url like 'git@github.com:/yaml-spec.git'" + @exit 1 +endif + ( \ + cd $(GH_PAGES) && \ + perl -pi -e "s/spec/$(FORK_USER_ID)-spec/" CNAME && \ + git add -A . && \ + git commit -m 'Stage' && \ + git push -f fork gh-pages \ + ) || true + @echo + @echo "Staged: http://$(FORK_USER_ID)-$(PUBLISH_CNAME)" + @echo publish: build +ifneq ($(shell git rev-parse --abbrev-ref HEAD),main) + @echo '*** Error' + @echo "You may only 'publish' from the 'main' branch" + @exit 1 +endif ( \ - cd $(PAGES) && \ + cd $(GH_PAGES) && \ git add -A . && \ git commit -m 'Publish' && \ - git push \ - ) + git push -f \ + ) || true + @echo + @echo "Published: https://$(PUBLISH_CNAME)" + @echo # Remove generated files to force rebuild: force: - $(call clean,$(PAGES)) - rm -fr $(STAGE) + $(call clean,$(GH_PAGES)) + rm -fr $(SITE) # Also touch the original spec-1.2 html: force-all: force make -C $(SPEC) force clean: - rm -fr $(STAGE) $(PAGES) + rm -fr $(SITE) $(GH_PAGES) $(SPEC)/spec.md: $(INPUT_HTML) make -C $(SPEC) spec.md YAML_SPEC_DIR= -$(STAGE)/%: % $(STAGE) +$(SITE)/%: % $(SITE) cp -r $< $@ -$(STAGE)/%: jekyll/% $(STAGE) +$(SITE)/%: jekyll/% $(SITE) cp -r $< $@ -$(STAGE)/%.md: %.md $(STAGE) - cat $< > $@ +$(SITE)/%.md: $(ROOT) %.md + mkdir -p $(dir $@) + render-markdown $^ > $@ -$(STAGE)/%.md: %.md.yaml $(SPEC)/%.md $(LINKS) $(STAGE) - render-spec-markdown $^ > $@ +$(SITE)/%.md: $(ROOT) $(SPEC)/%.md $(SPEC_LINKS) + mkdir -p $(dir $@) + render-markdown $^ > $@ cp $@ $(WORK)/ -$(STAGE)/spec-plain.md: spec-plain.md.yaml $(SPEC)/spec.md $(LINKS) $(STAGE) - render-spec-markdown $^ > $@ +$(SITE)/doc/%.md: $(ROOT) $(DOC)/%.md + mkdir -p $(dir $@) + render-markdown $^ > $@ + +$(SITE)/story/%.md: $(ROOT) $(STORY)/%.md + mkdir -p $(dir $@) + render-markdown $^ > $@ -$(STAGE)/%.png: $(SPEC)/img/%.png +$(SITE)/%.png: $(SPEC)/img/%.png cp $< $@ -$(STAGE)/%.css: %.css $(STAGE) +$(SITE)/%.css: %.css $(SITE) cp -r $< $@ -$(STAGE): $(SPEC_MD_HTML) $(WORK) +$(SITE): $(SPEC_MD_HTML) $(WORK) mkdir -p $@ cp $< $@/ cp $(WORK)/1.2/* $@/ -$(STAGE)/img: $(SPEC)/img +$(SITE)/img: $(SPEC)/img mkdir -p $@ cp -r $ $@ -$(PAGES): +$(GH_PAGES): -git branch --track gh-pages origin/gh-pages git worktree add -f $@ gh-pages diff --git a/www/ReadMe.md b/www/ReadMe.md index 500070c9..8152e15b 100644 --- a/www/ReadMe.md +++ b/www/ReadMe.md @@ -1,12 +1,115 @@ -YAML Spec as Markdown -===================== +YAML Spec Website Generation +============================ -This branch will generate the YAML spec as Markdown in the file `index.md`, -from the `spec-1.2` branch. +This directory is responsible for publishing the content of this (yaml-spec) +repository to . -Run `make build` to build it. +# Build System -The spec markdown can be viewed in the browser here: -. +Building, testing and publishing the website content is controlled by the +Makefile. +The Makefile supports: -It is also rendered by GitHub pages as . +* `make publish` + + Build and publish the content to . + +* `make serve` + + Build and serve locally to . + +* `make build` + + Build the site content into a finalized `./_gh-pages/` directory. + +* `make site` + + Gather the site content into a `./_site/` Jekyll source directory. + +* `make shell` + + Open a shell in the `github-pages` Docker container that builds the website + content. + +* `make force ...` + + The force rule will make sure everthing is rebuilt from scratch. + +* `make clean` + + Remove generated files. + +# Prerequisite Software + +The build system uses various open source software. + +## Required + +At a minimum you'll need: + +* `make` + + Of course. + +* `bash` + + Required to be installed on you system. + Not required to be the interactive shell you are using. + +* `docker` + + Everything else is encapsulated in Docker images. + If you have the required components installed locally they will be used, + otherwise `docker` will be invoked. + Docker is required for some complicated steps. + +## Optional + +Even though everything can be run by Docker, some build steps will run faster +with these local components installed. + +* Perl +* Perl CPAN modules: + * `YAML::PP` + * `XXX` +* NodeJS +* Node NPM modules: + * CoffeeScript (`npm install -g coffeescript`) + * `ingy-prelude` + * `turndown` + * `domino` + * `smartwrap` + +# Build Process + +This system is made out of Markdown, YAML, SCSS and images. +It is currently using Jekyll to build the final result. + +It gathers all the content in various directories throughout the repository and +puts them into the `./_site/` directory in a standard Jekyll layout. + +The intent is to not tie things too close to Jekyll or any other build system. + +The Jekyll build system is captured in the `github-pages` Docker image. +It is the same build process that GitHub Pages uses when you push Jekyll +content to it. +It builds the final HTML/CSS/JavaScript into the `./_gh-pages/` directory. +The `_gh-pages` content is pushed directly to the `gh-pages` branch of the +repository, and in turn served as . +No further Jekyll processing happens on the GitHub side after pushing. + +The Markdown files in the repository get preprocessed into a more complicated +but less readable Markdown form and put into `_site` directory. +For instance the post-processed forms might have a lot more Markdown HTML in +them. + +The intent is to: +* Keep the sources as simple and clean as possible. +* Have the Markdown files render pretty good in the GitHub views of them. +* Introduce a few powerful but not Markdown syntaxes. +* Have the final result on the website be amazing. + +We only introduce new syntax when absolutely necessary and try to keep it +minimal and non-disruptive. +Sometimes we use `` Markdown HTML comments so you don't see it in +the standard Makrdown renderings. diff --git a/www/index.md b/www/index.md index ddfbdfb0..877449cb 100644 --- a/www/index.md +++ b/www/index.md @@ -1,8 +1,14 @@ ---- -layout: default ---- YAML Specification Development ============================== -* [YAML Spec from Markdown](/spec.html) -* [Spec to Markdown Review](/review.html) +* [The YAML 1.x Spec](/spec) (in Development) + * [Spec to Markdown Review](/review) +* [YAML Developers Documentation](/doc/) +* [YAML Development Stories](/story/) + + diff --git a/www/spec-plain.md.yaml b/www/spec-plain.md similarity index 63% rename from www/spec-plain.md.yaml rename to www/spec-plain.md index b002ee78..66b6b5d9 100644 --- a/www/spec-plain.md.yaml +++ b/www/spec-plain.md @@ -1,4 +1,5 @@ --- layout: default plain: true +source: spec/spec.md --- diff --git a/www/spec.md.yaml b/www/spec.md.yaml deleted file mode 100644 index 0d6905dc..00000000 --- a/www/spec.md.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -layout: default ---- From 78dc8f2b85e19ae91678ba64e1fc05f5486365b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Mon, 21 Jun 2021 12:21:57 -0700 Subject: [PATCH 2/3] Fix sentence error reported by @perlpunk --- doc/glossary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/glossary.md b/doc/glossary.md index 10425985..88e24d30 100644 --- a/doc/glossary.md +++ b/doc/glossary.md @@ -94,8 +94,8 @@ meanings. * DOM - A DOM is an information state that is a tree of created by a composer or a - representer and consumed by a constructor or a serializer. + A DOM is an information state that is a tree of nodes created by a composer + or a representer and consumed by a constructor or a serializer. A DOM may also be consumed directly by an application. A DOM API can offer a lot more information and processing options than a native data structure. From 086338d6f2d8428fbd2723e86f8f1ce730c0ac1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Tue, 22 Jun 2021 07:43:18 -0700 Subject: [PATCH 3/3] Add support for publishing RFCs to web site --- rfc/RFC-0003.md | 8 +-- rfc/index.md | 13 ++++ tool/bin/render-markdown | 2 + tool/lib/MarkdownParser.pm | 120 ++++++++++++++++++++++++++++++++++++ tool/lib/render-markdown.pl | 114 +--------------------------------- www/Makefile | 13 ++++ www/index.md | 3 +- 7 files changed, 156 insertions(+), 117 deletions(-) create mode 100644 rfc/index.md create mode 100644 tool/lib/MarkdownParser.pm diff --git a/rfc/RFC-0003.md b/rfc/RFC-0003.md index 99d9307f..5d762a11 100644 --- a/rfc/RFC-0003.md +++ b/rfc/RFC-0003.md @@ -49,14 +49,14 @@ and for which any subsequent characters are from: ``` These categories are defined in terms of Unicode code point properties: -- `ID_Start`, `ID_Continue`: As described in +* `ID_Start`, `ID_Continue`: As described in [Unicode Standard Annex #31](http://www.unicode.org/reports/tr31/), "Unicode Identifier and Pattern Syntax". `ID_Continue` is a superset of `ID_Start` which also includes `Nd` and `Pc` characters, among others. -- `Nd`: Number, decimal digit -- `Pc`: Punctuation, connector (includes `_`) -- `Pd`: Punctuation, dash (includes `-`) +* `Nd`: Number, decimal digit +* `Pc`: Punctuation, connector (includes `_`) +* `Pd`: Punctuation, dash (includes `-`) ## Explanation diff --git a/rfc/index.md b/rfc/index.md new file mode 100644 index 00000000..a6a99a09 --- /dev/null +++ b/rfc/index.md @@ -0,0 +1,13 @@ +YAML Specification RFCs +======================= + +Changes to the YAML specification are made using an RFC process. + +## Current RFCs + +* [RFC-0001 — Empty-scalars not allowed as keys](RFC-0001) +* [RFC-0002 — Documents after the first require a document-start-indicator](RFC-0002) +* [RFC-0003 — Only ASCII-word-characters and dash are allowed in anchor names](RFC-0003) +* [RFC-0004 — Maximum of 8 spaces per indentation level](RFC-0004) +* [RFC-0005 — Directives belong to the stream and not a document](RFC-0005) +* [RFC-0006 — Bare documents of flow collections may be separated by a newline](RFC-0006) diff --git a/tool/bin/render-markdown b/tool/bin/render-markdown index b4f3a59b..72a2010a 100755 --- a/tool/bin/render-markdown +++ b/tool/bin/render-markdown @@ -12,4 +12,6 @@ check() ( ' || return 1 ) +export PERL5LIB=$YAML_SPEC_ROOT/tool/lib + run "$@" diff --git a/tool/lib/MarkdownParser.pm b/tool/lib/MarkdownParser.pm new file mode 100644 index 00000000..bdf9cff8 --- /dev/null +++ b/tool/lib/MarkdownParser.pm @@ -0,0 +1,120 @@ +use v5.18; +package MarkdownParser; + +my $re_pre = qr/ + ```\n + (?:.*\n)+? + ```\n +/x; +my $re_legend = qr/ + \*\*Legend:\*\*\n + (?:\*\ .*\n)+ +/x; + +sub parse_sections { + ($_) = @_; + + my @s; + while ($_) { + # Skip blank lines: + s/\A\n+// and next; + + # Headings: + s/\A(\S.*\n)===+\n\n+// + and push @s, {heading => "# $1"} and next; + + s/\A(#{1,5} .*\n)\n+// + and push @s, {heading => $1} and next; + + # Example section: + s/\A + (\*\*Example\ \# [\s\S]*?\*\*)\n + ($re_pre) + ($re_pre)? + ($re_legend)? + \n+ + //x + and push @s, {example => [$1, $2, $3, $4]} and next; + + # Lists: + s/\A + ( + (::.*\n)? + (?: (?:\*|\d+\.) \ \S.*\n) + (?: + \n? + (?: + (?: \ * (?:\*|\d+\.) \ \S.*\n) | + (?: \ + \S.*\n) | + ) + )* + (?:\{:\.\S+\}\n)? + ) + \n+ + //x + and push @s, {ul => $1} and next; + + # Paragraphs + s/\A + ( + (?: + (?: + \*\* | + -1\ | + \[ [\w\"\*] | + [\w\"\(\`] + ) + .*\n + )+ + ) + (?: + (?=```|\*\ ) | + \z | + \n+ + ) + //x + and push @s, {p => $1} and next; + + s/\A(----\n)\n+// + and push @s, {hr => $1} and next; + + s/\A\::toc\n\n+// + and push @s, {toc => 1} and next; + + s/\A\::index\n// + and push @s, {index => 1} and next; + + s/\A((?:>.*\n)+)\n+// + and push @s, {indent => $1} and next; + + s/\A((?s:```.+?\n```\n))\n+// + and push @s, {pre => $1} and next; + + s/\A(!\[.*\.png\)\n)\s+// + and push @s, {img => $1} and next; + + s/\A(\* .*\n(?: \S.*\n|\n(?= \S))+)\n+// + and push @s, {ul => $1} and next; + + s/\A(\`\w.*\n)\n+// + and push @s, {p => $1} and next; + + s/\A((?:\| .*\n)+)\n+// + and push @s, {table => $1} and next; + + s/\A((?:\* .*\n)(?: .*\n|\n(?= ))+)\n+// + and push @s, {ul => $1} and next; + + s/\A(\n)\n*// + and push @s, {comment => $1} and next; + + s/((?:.*\n){20})(?s:.*)/$1/; + require XXX; + XXX::WWW(\@s); + die "*** ERROR ***\nMarkdownParser failed at this point:\n$_\n*** EOF ***\n"; + } + + return \@s; +} + +1; diff --git a/tool/lib/render-markdown.pl b/tool/lib/render-markdown.pl index 922ca1dc..ff588c5a 100644 --- a/tool/lib/render-markdown.pl +++ b/tool/lib/render-markdown.pl @@ -1,8 +1,8 @@ #!/usr/bin/env perl use v5.18; -# use utf8; +use MarkdownParser; use YAML::PP; use XXX; @@ -13,7 +13,7 @@ sub main { my ($front, $markdown, $link_map) = read_files(@_); - my $parsed = parse_sections($markdown); + my $parsed = MarkdownParser::parse_sections($markdown); set_vars(); make_link_index($parsed, $link_map); @@ -32,116 +32,6 @@ sub main { } #------------------------------------------------------------------------------ -my $re_pre = qr/ - ```\n - (?:.*\n)+? - ```\n -/x; -my $re_legend = qr/ - \*\*Legend:\*\*\n - (?:\*\ .*\n)+ -/x; - -sub parse_sections { - ($_) = @_; - - my @s; - while ($_) { - # Headings: - s/\A(\S.*\n)===+\n\n+// - and push @s, {heading => "# $1"} and next; - - s/\A(#{1,5} .*\n)\n+// - and push @s, {heading => $1} and next; - - # Example section: - s/\A - (\*\*Example\ \# [\s\S]*?\*\*)\n - ($re_pre) - ($re_pre)? - ($re_legend)? - \n+ - //x - and push @s, {example => [$1, $2, $3, $4]} and next; - - s/\A - ( - (::.*\n)? - (?: - (?:\*\ \S.*\n) - (?: - (?: - \n - (\ \ \S.*\n)+ - )+ - \n? - )? - (?:\ \ \*\ .*\n)* - )+ - (?:\{:\.\S+\}\n)? - ) - \n+ - //x - and push @s, {ul => $1} and next; - - s/\A - ( - (?: - (?:\*\*|-1\ |\[[\w\"\*]|[\w\"\(]) - .*\n - )+ - ) - (?: - (?=```) | - \z | - \n+ - ) - //x - and push @s, {p => $1} and next; - - s/\A(----\n)\n+// - and push @s, {hr => $1} and next; - - s/\A\::toc\n\n+// - and push @s, {toc => 1} and next; - - s/\A\::index\n// - and push @s, {index => 1} and next; - - s/\A((?:>.*\n)+)\n+// - and push @s, {indent => $1} and next; - - s/\A((?s:```.+?\n```\n))\n+// - and push @s, {pre => $1} and next; - - s/\A(!\[.*\.png\)\n)\s+// - and push @s, {img => $1} and next; - - s/\A(\* .*\n(?: \S.*\n|\n(?= \S))+)\n+// - and push @s, {ul => $1} and next; - - s/\A(\`\w.*\n)\n+// - and push @s, {p => $1} and next; - - s/\A((?:\| .*\n)+)\n+// - and push @s, {table => $1} and next; - - s/\A((?:\* .*\n)(?: .*\n|\n(?= ))+)\n+// - and push @s, {ul => $1} and next; - - s/\A(\n)\n*// - and push @s, {comment => $1} and next; - - s/\n+\z// and next; - - s/((?:.*\n){20})(?s:.*)/$1/; - WWW(\@s); - die "*** ERROR ***\nParse failed at this point:\n$_\n*** EOF ***\n"; - } - - return \@s; -} - #------------------------------------------------------------------------------ sub fmt_heading { set_dates(); diff --git a/www/Makefile b/www/Makefile index f58949a8..f57b61e3 100644 --- a/www/Makefile +++ b/www/Makefile @@ -5,6 +5,7 @@ PUBLISH_CNAME := spec.yaml.io SPEC12 := $(ROOT)/1.2 SPEC := $(ROOT)/spec +RFC := $(ROOT)/rfc DOC := $(ROOT)/doc STORY := $(ROOT)/story WORK := $(ROOT)/work @@ -16,6 +17,13 @@ SPEC_IMAGE := $(shell ls $(SPEC)/img/) SPEC_MD_HTML := $(WORK)/markdown.html INPUT_HTML := $(WORK)/spec-1.2.html +RFC_MD := $(shell \ + find $(RFC) -name '*.md' | \ + sort | \ + grep -Ev '(ReadMe.md|0000)' \ +) +RFC_MD := $(RFC_MD:$(ROOT)/%=%) + DOC_MD := $(shell \ find $(DOC) -name '*.md' | \ grep -v ReadMe.md \ @@ -53,6 +61,7 @@ SITE_FILES := \ $(WWW_FILES) \ spec.md \ $(SPEC_IMAGE) \ + $(RFC_MD) \ $(DOC_MD) \ $(STORY_MD) \ Gemfile \ @@ -153,6 +162,10 @@ $(SITE)/%.md: $(ROOT) $(SPEC)/%.md $(SPEC_LINKS) render-markdown $^ > $@ cp $@ $(WORK)/ +$(SITE)/rfc/%.md: $(ROOT) $(RFC)/%.md + mkdir -p $(dir $@) + render-markdown $^ > $@ + $(SITE)/doc/%.md: $(ROOT) $(DOC)/%.md mkdir -p $(dir $@) render-markdown $^ > $@ diff --git a/www/index.md b/www/index.md index 877449cb..6f44c22f 100644 --- a/www/index.md +++ b/www/index.md @@ -1,8 +1,9 @@ YAML Specification Development ============================== -* [The YAML 1.x Spec](/spec) (in Development) +* [The YAML 1.x Specification](/spec) (in Development) * [Spec to Markdown Review](/review) +* [YAML Specification RFCs](/rfc/) * [YAML Developers Documentation](/doc/) * [YAML Development Stories](/story/)