diff --git a/.gitignore b/.gitignore index 0d12393..3085483 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ content/notebooks data/bibliography.json .sass-cache .ipynb_checkpoints +.hugo_build.lock +bin/nbconvert/nbconvert \ No newline at end of file diff --git a/Makefile b/Makefile index 06edc3e..1599603 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,18 @@ serve: prepare serve-drafts: prepare hugo server -D -prepare: convert +prepare: experience bibliography notebooks + +experience: bin/sort-experience.ts data/experience.json -convert: +bibliography: bin/biblio.ts assets/bib/bibliography.bib -m assets/bib/members.json -o data/bibliography.json - go run bin/nbconvert.go +notebooks: nbconvert + bin/nbconvert/nbconvert + +nbconvert: + +make -C bin/nbconvert + +.PHONY: prepare experience bibliography notebooks diff --git a/assets/sass/39alpha/base.scss b/assets/sass/39alpha/base.scss index cbeb83a..075cb99 100644 --- a/assets/sass/39alpha/base.scss +++ b/assets/sass/39alpha/base.scss @@ -36,6 +36,7 @@ figure > img { } figcaption { + margin-top: $spacing-unit / 2; font-size: $small-font-size; } @@ -100,7 +101,7 @@ blockquote { pre, code { @include relative-font-size(0.9375); - border: 1px solid $grey-color-light; + border: 1px solid $grey-color; border-radius: 3px; background-color: #eef; } diff --git a/assets/sass/39alpha/layout.scss b/assets/sass/39alpha/layout.scss index 33dade4..67d6471 100644 --- a/assets/sass/39alpha/layout.scss +++ b/assets/sass/39alpha/layout.scss @@ -304,7 +304,9 @@ .post-content, .notebook-content { margin-bottom: $spacing-unit; - h2 { + h1 { + text-align: center; + @include relative-font-size(2); @include media-query($on-laptop) { @@ -312,7 +314,9 @@ } } - h3 { + h2 { + text-align: center; + @include relative-font-size(1.625); @include media-query($on-laptop) { @@ -320,13 +324,26 @@ } } - h4 { + h3 { + text-align: center; + @include relative-font-size(1.25); @include media-query($on-laptop) { @include relative-font-size(1.125); } } + + h4 { + text-align: center; + font-style: italic; + + @include relative-font-size(1.125); + + @include media-query($on-laptop) { + @include relative-font-size(1); + } + } } .notebook-content { @@ -347,6 +364,10 @@ } } +.observablehq { + margin-bottom: $spacing-unit / 2; +} + .giving { width: 100%; height: auto; @@ -533,12 +554,14 @@ } } -#code-hider { +.code-hider { + color: $brand-color; text-align: right; padding-bottom: 10px; + user-select: none; span { - color: $brand-color; + margin-left: $spacing-unit / 4; text-decoration: none; &:hover { diff --git a/assets/sass/39alpha/syntax-highlighting.scss b/assets/sass/39alpha/syntax-highlighting.scss index d6b3aca..3229805 100644 --- a/assets/sass/39alpha/syntax-highlighting.scss +++ b/assets/sass/39alpha/syntax-highlighting.scss @@ -1,64 +1,89 @@ .chroma { - background: #eef; - @extend %vertical-rhythm; + color: $text-color; + background-color: $background-color; + @extend %vertical-rhythm; - .c { color: #998; font-style: italic } // Comment - .err { color: #a61717; background-color: #e3d2d2 } // Error - .k { font-weight: bold } // Keyword - .o { font-weight: bold } // Operator - .cm { color: #998; font-style: italic } // Comment.Multiline - .cp { color: #999; font-weight: bold } // Comment.Preproc - .c1 { color: #998; font-style: italic } // Comment.Single - .cs { color: #999; font-weight: bold; font-style: italic } // Comment.Special - .gd { color: #000; background-color: #fdd } // Generic.Deleted - .gd .x { color: #000; background-color: #faa } // Generic.Deleted.Specific - .ge { font-style: italic } // Generic.Emph - .gr { color: #a00 } // Generic.Error - .gh { color: #999 } // Generic.Heading - .gi { color: #000; background-color: #dfd } // Generic.Inserted - .gi .x { color: #000; background-color: #afa } // Generic.Inserted.Specific - .go { color: #888 } // Generic.Output - .gp { color: #555 } // Generic.Prompt - .gs { font-weight: bold } // Generic.Strong - .gu { color: #aaa } // Generic.Subheading - .gt { color: #a00 } // Generic.Traceback - .kc { font-weight: bold } // Keyword.Constant - .kd { font-weight: bold } // Keyword.Declaration - .kp { font-weight: bold } // Keyword.Pseudo - .kr { font-weight: bold } // Keyword.Reserved - .kt { color: #458; font-weight: bold } // Keyword.Type - .m { color: #099 } // Literal.Number - .s { color: #d14 } // Literal.String - .na { color: #008080 } // Name.Attribute - .nb { color: #0086B3 } // Name.Builtin - .nc { color: #458; font-weight: bold } // Name.Class - .no { color: #008080 } // Name.Constant - .ni { color: #800080 } // Name.Entity - .ne { color: #900; font-weight: bold } // Name.Exception - .nf { color: #900; font-weight: bold } // Name.Function - .nn { color: #555 } // Name.Namespace - .nt { color: #000080 } // Name.Tag - .nv { color: #008080 } // Name.Variable - .ow { font-weight: bold } // Operator.Word - .w { color: #bbb } // Text.Whitespace - .mf { color: #099 } // Literal.Number.Float - .mh { color: #099 } // Literal.Number.Hex - .mi { color: #099 } // Literal.Number.Integer - .mo { color: #099 } // Literal.Number.Oct - .sb { color: #d14 } // Literal.String.Backtick - .sc { color: #d14 } // Literal.String.Char - .sd { color: #d14 } // Literal.String.Doc - .s2 { color: #d14 } // Literal.String.Double - .se { color: #d14 } // Literal.String.Escape - .sh { color: #d14 } // Literal.String.Heredoc - .si { color: #d14 } // Literal.String.Interpol - .sx { color: #d14 } // Literal.String.Other - .sr { color: #009926 } // Literal.String.Regex - .s1 { color: #d14 } // Literal.String.Single - .ss { color: #990073 } // Literal.String.Symbol - .bp { color: #999 } // Name.Builtin.Pseudo - .vc { color: #008080 } // Name.Variable.Class - .vg { color: #008080 } // Name.Variable.Global - .vi { color: #008080 } // Name.Variable.Instance - .il { color: #099 } // Literal.Number.Integer.Long + .x { } + .err { color: #a61717; background-color: #e3d2d2 } + .cl { } + .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; } + .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; } + .hl { background-color: #ffffcc } + .lnt { white-space: pre; user-select: none; margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #686868 } + .ln { white-space: pre; user-select: none; margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #686868 } + .line { display: flex; } + .k { color: #6ab825; font-weight: bold } + .kc { color: #6ab825; font-weight: bold } + .kd { color: #6ab825; font-weight: bold } + .kn { color: #6ab825; font-weight: bold } + .kp { color: #6ab825 } + .kr { color: #6ab825; font-weight: bold } + .kt { color: #6ab825; font-weight: bold } + .n { } + .na { color: #bbbbbb } + .nb { color: #24909d } + .bp { } + .nc { color: #447fcf; text-decoration: underline } + .no { color: #40ffff } + .nd { color: #ffa500 } + .ni { } + .ne { color: #bbbbbb } + .nf { color: #447fcf } + .fm { } + .nl { } + .nn { color: #447fcf; text-decoration: underline } + .nx { } + .py { } + .nt { color: #6ab825; font-weight: bold } + .nv { color: #40ffff } + .vc { } + .vg { } + .vi { } + .vm { } + .l { } + .ld { } + .s { color: #ed9d13 } + .sa { color: #ed9d13 } + .sb { color: #ed9d13 } + .sc { color: #ed9d13 } + .dl { color: #ed9d13 } + .sd { color: #ed9d13 } + .s2 { color: #ed9d13 } + .se { color: #ed9d13 } + .sh { color: #ed9d13 } + .si { color: #ed9d13 } + .sx { color: #ffa500 } + .sr { color: #ed9d13 } + .s1 { color: #ed9d13 } + .ss { color: #ed9d13 } + .m { color: #3677a9 } + .mb { color: #3677a9 } + .mf { color: #3677a9 } + .mh { color: #3677a9 } + .mi { color: #3677a9 } + .il { color: #3677a9 } + .mo { color: #3677a9 } + .o { } + .ow { color: #6ab825; font-weight: bold } + .p { } + .c { color: #999999; font-style: italic } + .ch { color: #999999; font-style: italic } + .cm { color: #999999; font-style: italic } + .c1 { color: #999999; font-style: italic } + .cs { color: #e50808; background-color: #520000; font-weight: bold } + .cp { color: #cd2828; font-weight: bold } + .cpf { color: #cd2828; font-weight: bold } + .g { } + .gd { color: #d22323 } + .ge { font-style: italic } + .gr { color: #d22323 } + .gh { color: #ffffff; font-weight: bold } + .gi { color: #589819 } + .go { color: #cccccc } + .gp { color: #aaaaaa } + .gs { font-weight: bold } + .gu { color: #ffffff; text-decoration: underline } + .gt { color: #d22323 } + .gl { text-decoration: underline } + .w { color: #666666 } } diff --git a/bin/extract-assets.py b/bin/extract-assets.py deleted file mode 100644 index 6f1644e..0000000 --- a/bin/extract-assets.py +++ /dev/null @@ -1,33 +0,0 @@ -import argparse -import nbconvert -import os.path - - -def notebook_dir(fname): - return os.path.join("content", os.path.dirname(fname)) - - -def extract_from_notebook(fname, outdir=None): - if outdir is None: - outdir = notebook_dir(fname) - - _, resources = nbconvert.MarkdownExporter().from_filename(fname) - if "outputs" in resources: - for resource, data in resources["outputs"].items(): - path = os.path.join(outdir, resource) - print("{}:{} → {}".format(fname, resource, path)) - with open(path, "wb") as handle: - handle.write(data) - - -if __name__ == '__main__': - description = 'Extract output assets from a Jupyter notebook' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--notebook', metavar='N', type=str, nargs=1, - help='path to notebook to extract') - parser.add_argument('--outdir', metavar='O', type=str, nargs='?', - help='path into which to output extracted assets') - - args = parser.parse_args() - - extract_from_notebook(args.notebook[0], args.outdir) diff --git a/bin/jupyter-convert.py b/bin/jupyter-convert.py deleted file mode 100644 index d3457cc..0000000 --- a/bin/jupyter-convert.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys, os.path -from nbconvert import MarkdownExporter - -if len(sys.argv) == 1: - raise(RuntimeError('at least one juptyer notebook filename must be provided')) - -for notebook in sys.argv[1:]: - body, _ = MarkdownExporter().from_filename(notebook) - print(body) diff --git a/bin/nbconvert.go b/bin/nbconvert.go deleted file mode 100644 index ec330ad..0000000 --- a/bin/nbconvert.go +++ /dev/null @@ -1,278 +0,0 @@ -package main - -import ( - "archive/zip" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "text/template" -) - -const ( - checkpoints = ".ipynb_checkpoints" - notebook_path = "notebooks" - notebook_name = "notebook.ipynb" - draft_notebook_name = "notebook_draft.ipynb" - markdown_name = "index.md" - jupyter_convert = "bin/jupyter-convert.py" - extract_assets = "bin/extract-assets.py" -) - -var content_root = filepath.Join("content", "notebooks") -var markdown_template = template.Must(template.New("Notebook").Parse(`--- -title: {{ .Title }} -draft: {{ .Draft }} ---- -{{ .Content }}`)) - -func PythonExe() string { - python_exe, err := exec.LookPath("python3") - if err != nil { - python_exe, err = exec.LookPath("python") - if err != nil { - log.Fatalf("Could not find `python` or `python3` in your PATH; perhaps you need to install it?") - } - } - return python_exe -} - -func CopyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, in) - if err != nil { - return err - } - return out.Close() -} - -func WriteZip(notebook, content_dir string) error { - notebook_dir := filepath.Dir(notebook) - - zipfile := filepath.Join(content_dir, "notebook.zip") - outFile, err := os.Create(zipfile) - if err != nil { - return err - } - defer outFile.Close() - - w := zip.NewWriter(outFile) - - if err := AddFiles(w, notebook_dir, ""); err != nil { - return err - } - - if err = w.Close(); err != nil { - return err - } - - return nil -} - -func AddFiles(w *zip.Writer, basePath, baseInZip string) error { - files, err := ioutil.ReadDir(basePath) - if err != nil { - return err - } - - for _, file := range files { - if !file.IsDir() { - data, err := ioutil.ReadFile(filepath.Join(basePath, file.Name())) - if err != nil { - return err - } - - f, err := w.Create(filepath.Join(baseInZip, file.Name())) - if err != nil { - return err - } - - _, err = f.Write(data) - if err != nil { - return err - } - } else if file.IsDir() { - newBase := filepath.Join(basePath, file.Name()) - AddFiles(w, newBase, filepath.Join(baseInZip, file.Name())) - } - } - - return nil -} - -func FindNotebooks() (files []string, err error) { - err = filepath.Walk(notebook_path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - basename := filepath.Base(path) - if info.Mode().IsRegular() && (basename == notebook_name || basename == draft_notebook_name) { - files = append(files, path) - } - return nil - }) - return -} - -type Notebook struct { - Path string - Title string - Draft bool - Content string - Assets []string -} - -func ReadNotebook(path string) (*Notebook, error) { - dir := filepath.Dir(path) - - title := filepath.Base(dir) - draft := filepath.Base(path) == draft_notebook_name - - cmd := exec.Command(PythonExe(), jupyter_convert, path) - - content, err := cmd.Output() - if err != nil { - return nil, err - } - - re := regexp.MustCompile(`^\s*#[^\n]+`) - header := re.Find(content) - if (header != nil ) { - re = regexp.MustCompile(`#`) - i := re.FindIndex(header)[0] + 1 - for header[i] == ' ' { - i += 1 - } - title = string(header[i:len(header)]) - } - - assets := []string{} - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - basename := filepath.Base(path) - if info.Mode().IsRegular() && basename != notebook_name && basename != draft_notebook_name { - for dir := filepath.Dir(path); dir != "/" && dir != "." && dir != ".." && dir != ""; dir = filepath.Dir(dir) { - if filepath.Base(dir) == checkpoints { - return nil - } - } - assets = append(assets, path) - } - return nil - }) - - if err != nil { - return nil, err - } - - return &Notebook{ path, title, draft, string(content), assets }, nil -} - -func (nb *Notebook) CopyAssets(dir string) error { - for _, asset := range nb.Assets { - dest_dir, err := ContentDir(asset) - if err != nil { - return err - } - dest := filepath.Join(dest_dir, filepath.Base(asset)) - fmt.Printf("%s → %s\n", asset, dest) - if err = os.MkdirAll(dest_dir, os.ModeDir|os.ModePerm); err != nil { - return fmt.Errorf("cannot create asset directory %q: %v", dest_dir, err) - } - - CopyFile(asset, dest) - } - return nil -} - -func (nb *Notebook) ExtractAssets(dir string) error { - cmd := exec.Command(PythonExe(), extract_assets, "--notebook", nb.Path, "--outdir", dir) - - content, err := cmd.Output() - if err != nil { - return err - } - fmt.Print(string(content)) - return nil -} - -func ContentDir(path string) (string, error) { - relpath, err := filepath.Rel(notebook_path, path) - if err != nil { - return "", err - } - relpath = filepath.Dir(relpath) - return filepath.Join(content_root, relpath), nil -} - -func main() { - if err := os.RemoveAll(content_root); err != nil { - log.Fatalf("error removing notebooks directory %q: %v\n", content_root, err) - } - - notebooks, err := FindNotebooks() - if err != nil { - log.Fatalf("error finding notebooks; %v\n", err) - } - - if err = os.MkdirAll(content_root, os.ModeDir|os.ModePerm); err != nil { - log.Fatalf("error creating notebook content directory: %v\n", err) - } - - if _, err := os.Create(filepath.Join(content_root, "_index.md")); err != nil { - log.Fatalf("error creating the notebook listing page: %v\n", err) - } - - for _, notebook := range notebooks { - content_dir, err := ContentDir(notebook) - if err != nil { - log.Fatalf("error determining content directory for notebook %q: %v\n", notebook, err) - } - fmt.Printf("%s → %s\n", notebook, content_dir) - nb, err := ReadNotebook(notebook) - if err != nil { - log.Fatalf("error reading notebook %q: %v\n", notebook, err) - } - - if err = os.MkdirAll(content_dir, os.ModeDir|os.ModePerm); err != nil { - log.Fatalf("error creating notebook directory: %v\n", err) - } - - notebook_filename := filepath.Join(content_dir, markdown_name) - if file, err := os.Create(notebook_filename); err != nil { - log.Fatalf("error opening file %q: %v\n", notebook_filename, err) - } else { - if err = markdown_template.Execute(file, nb); err != nil { - log.Fatalf("error executing template: %v\n", err) - } - if err = file.Close(); err != nil { - log.Fatalf("error closing file %q: %v\n", notebook_filename, err) - } - } - nb.CopyAssets(content_dir) - nb.ExtractAssets(content_dir) - - if err = WriteZip(notebook, content_dir); err != nil { - log.Fatalf("error writing notebook zip: %v\n", err) - } - } -} diff --git a/bin/nbconvert/Makefile b/bin/nbconvert/Makefile new file mode 100644 index 0000000..0d58aaf --- /dev/null +++ b/bin/nbconvert/Makefile @@ -0,0 +1,7 @@ +all: + go build + +clean: + rm nbconvert + +.PHONY: clean diff --git a/bin/nbconvert/go.mod b/bin/nbconvert/go.mod new file mode 100644 index 0000000..34b6db0 --- /dev/null +++ b/bin/nbconvert/go.mod @@ -0,0 +1,3 @@ +module github.com/39alpha/39alpharesearch.org/nbconvert + +go 1.19 diff --git a/bin/nbconvert/ipynb.go b/bin/nbconvert/ipynb.go new file mode 100644 index 0000000..7cbed8b --- /dev/null +++ b/bin/nbconvert/ipynb.go @@ -0,0 +1,37 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "path/filepath" +) + +type KernelSpec struct { + Language Language `json:"language"` +} + +type MetaData struct { + KernelSpec KernelSpec `json:"kernelspec"` +} + +type Ipynb struct { + MetaData MetaData `json:"metadata"` +} + +func IsIpynb(path string) bool { + return filepath.Ext(path) == ".ipynb" +} + +func InferIpynbLanguage(path string) (Language, error) { + var ipynb Ipynb + content, err := ioutil.ReadFile(path) + if err != nil { + return UnknownLang, err + } + + if err = json.Unmarshal(content, &ipynb); err != nil { + return UnknownLang, err + } + + return ipynb.MetaData.KernelSpec.Language, nil +} diff --git a/bin/nbconvert/julia.go b/bin/nbconvert/julia.go new file mode 100644 index 0000000..6e7c460 --- /dev/null +++ b/bin/nbconvert/julia.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" +) + +func JuliaExe() string { + julia, err := exec.LookPath("julia") + if err != nil { + log.Fatalf("Could not find `julia`; perhaps you need to install it?") + } + return julia +} + +type JuliaNotebook struct { + path string + assets []*Asset +} + +func (nb *JuliaNotebook) Path() string { + return nb.path +} + +func (nb *JuliaNotebook) Assets() []*Asset { + return nb.assets +} + +func (nb *JuliaNotebook) AddAsset(asset *Asset) []*Asset { + nb.assets = append(nb.assets, asset) + return nb.assets +} + +func (nb *JuliaNotebook) IsIgnoredAsset(path string) bool { + name := filepath.Base(path) + switch name { + case "Manifest.toml": + return true + case "Project.toml": + return true + default: + return false + } +} + +func (nb *JuliaNotebook) Instantiate() error { + if info, err := os.Stat(AssetPath(nb, "Project.toml")); err != nil || !info.Mode().IsRegular() { + return nil + } + + cmd := exec.Command(JuliaExe(), "--project=.", "-E", "using Pkg; Pkg.instantiate()") + + _, _, err := RunCommand(nb, cmd) + if err != nil { + return fmt.Errorf("failed to instantiate project %q: %v\n", nb.Path(), err) + } + + return nil +} + +func (nb *JuliaNotebook) Render() error { + if err := nb.Instantiate(); err != nil { + return err + } + return Quarto(nb) +} diff --git a/bin/nbconvert/main.go b/bin/nbconvert/main.go new file mode 100644 index 0000000..6f7c556 --- /dev/null +++ b/bin/nbconvert/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sync" +) + +const ( + checkpoints = ".ipynb_checkpoints" + notebook_path = "notebooks" + notebook_name = "notebook.ipynb" + quarto_name = "notebook.qmd" + markdown_name = "index.md" + generated_assets = "notebook_files" +) + +var content_root = filepath.Join("content", "notebooks") + +func main() { + if err := os.RemoveAll(content_root); err != nil { + log.Fatalf("error removing notebooks directory %q: %v\n", content_root, err) + } + + notebooks, err := FindNotebooks() + if err != nil { + log.Fatalf("error finding notebooks; %v\n", err) + } + + if err = os.MkdirAll(content_root, os.ModeDir|os.ModePerm); err != nil { + log.Fatalf("error creating notebook content directory: %v\n", err) + } + + if _, err := os.Create(filepath.Join(content_root, "_index.md")); err != nil { + log.Fatalf("error creating the notebook listing page: %v\n", err) + } + + var wg sync.WaitGroup + + for _, notebook := range notebooks { + wg.Add(1) + go func(nb Notebook) { + defer wg.Done() + if err := Execute(nb); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + }(notebook) + } + + wg.Wait() +} diff --git a/bin/nbconvert/notebook.go b/bin/nbconvert/notebook.go new file mode 100644 index 0000000..de00803 --- /dev/null +++ b/bin/nbconvert/notebook.go @@ -0,0 +1,306 @@ +package main + +import ( + "archive/zip" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" +) + +type Language string + +const ( + PythonLang Language = "python" + JuliaLang = "julia" + RLang = "r" + ObservableLang = "observable" + UnknownLang = "unknown" +) + +func isNotebook(path string, file os.FileInfo) bool { + basename := filepath.Base(path) + return file.Mode().IsRegular() && (basename == notebook_name || basename == quarto_name) +} + +type Asset struct { + path string + info os.FileInfo +} + +type Notebook interface { + Path() string + Assets() []*Asset + AddAsset(*Asset) []*Asset + IsIgnoredAsset(path string) bool + Render() error +} + +func BaseName(nb Notebook) string { + return filepath.Base(nb.Path()) +} + +func Directory(nb Notebook) string { + return filepath.Dir(nb.Path()) +} + +func AssetPath(nb Notebook, path string) string { + return filepath.Join(Directory(nb), path) +} + +func FindNotebooks() (notebooks []Notebook, err error) { + err = filepath.WalkDir(notebook_path, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + + if entry.IsDir() { + return nil + } + + info, err := entry.Info() + if err != nil { + return err + } + + if isNotebook(path, info) { + var nb Notebook + + lang, err := InferLanguage(path) + if err != nil { + return err + } + + switch lang { + case PythonLang: + nb = &PythonNotebook{path, []*Asset{}} + case JuliaLang: + nb = &JuliaNotebook{path, []*Asset{}} + case RLang: + nb = &RNotebook{path, []*Asset{}} + case ObservableLang: + nb = &ObservableNotebook{path, []*Asset{}} + default: + return fmt.Errorf("unable to infer language for notebook %q", path) + } + + if err = FindAssets(nb); err != nil { + return err + } + notebooks = append(notebooks, nb) + } + return nil + }) + return +} + +func InferLanguage(path string) (Language, error) { + if IsQuarto(path) { + return InferQuartoLanguage(path) + } + return InferIpynbLanguage(path) +} + +func FindAssets(nb Notebook) error { + dir := Directory(nb) + files, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, file := range files { + path := AssetPath(nb, file.Name()) + if nb.IsIgnoredAsset(path) { + continue + } + + info, err := file.Info() + if err != nil { + return err + } + + if isNotebook(path, info) || file.Name() == checkpoints { + continue + } + nb.AddAsset(&Asset{path, info}) + } + return nil +} + +func ContentDir(nb Notebook) (string, error) { + relpath, err := filepath.Rel(notebook_path, nb.Path()) + if err != nil { + return "", err + } + relpath = filepath.Dir(relpath) + return filepath.Join(content_root, relpath), nil + +} + +func CopyAssets(nb Notebook) error { + dest_dir, err := ContentDir(nb) + if err != nil { + return err + } + for _, asset := range nb.Assets() { + dest := filepath.Join(dest_dir, filepath.Base(asset.path)) + if err = os.MkdirAll(dest_dir, os.ModeDir|os.ModePerm); err != nil { + return fmt.Errorf("cannot create asset directory %q: %v", dest_dir, err) + } + + if asset.info.IsDir() { + CopyDirectory(asset.path, dest) + } else { + CopyFile(asset.path, dest) + } + } + return nil +} + +func GeneratedAssets(nb Notebook) []*Asset { + assets := []*Asset{} + + dir := Directory(nb) + paths := []string{ + filepath.Join(dir, markdown_name), + filepath.Join(dir, generated_assets), + } + + for _, path := range paths { + if info, err := os.Stat(path); err == nil { + assets = append(assets, &Asset{path, info}) + } + } + + return assets +} + +func MoveGeneratedAssets(nb Notebook) error { + dest_dir, err := ContentDir(nb) + if err != nil { + return err + } + for _, asset := range GeneratedAssets(nb) { + dest := filepath.Join(dest_dir, filepath.Base(asset.path)) + if err := os.Rename(asset.path, dest); err != nil { + return err + } + } + return nil +} + +func Cleanup(nb Notebook) error { + for _, asset := range GeneratedAssets(nb) { + if err := os.RemoveAll(asset.path); err != nil { + return err + } + } + + if IsQuarto(nb.Path()) { + return os.RemoveAll(AssetPath(nb, notebook_name)) + } + + return nil +} + +func WriteZip(nb Notebook) error { + notebook_dir := Directory(nb) + content_dir, err := ContentDir(nb) + if err != nil { + return nil + } + + zipfile := filepath.Join(content_dir, "notebook.zip") + outFile, err := os.Create(zipfile) + if err != nil { + return err + } + defer outFile.Close() + + w := zip.NewWriter(outFile) + + if err := AddFiles(w, notebook_dir, ""); err != nil { + return err + } + + if err = w.Close(); err != nil { + return err + } + + return nil +} + +func Execute(nb Notebook) error { + content_dir, err := ContentDir(nb) + if err != nil { + return fmt.Errorf("error determining content directory for notebook %q: %v\n", nb.Path(), err) + } + + if err = os.MkdirAll(content_dir, os.ModeDir|os.ModePerm); err != nil { + return fmt.Errorf("error creating notebook directory: %v\n", err) + } + + fmt.Printf("Executing notebook %q\n", nb.Path()) + + if err = Cleanup(nb); err != nil { + return fmt.Errorf("error cleaning up generated assets for notebook %q: %v\n", nb.Path(), err) + } + + if err = CopyAssets(nb); err != nil { + return fmt.Errorf("error copying assets to content directory for notebook %q: %v\n", nb.Path(), err) + } + + if err = nb.Render(); err != nil { + return fmt.Errorf("error converting notebook %q: %v\n", nb.Path(), err) + } + defer Cleanup(nb) + + if err = MoveGeneratedAssets(nb); err != nil { + return fmt.Errorf("error copying generated assets to content directory for notebook %q: %v\n", nb.Path(), err) + } + + if err = WriteZip(nb); err != nil { + return fmt.Errorf("error writing notebook zip for notebook %q: %v\n", nb.Path(), err) + } + + return nil +} + +func RunCommand(nb Notebook, cmd *exec.Cmd) ([]byte, []byte, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, nil, err + } + + cmd.Dir = filepath.Join(cwd, Directory(nb)) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, nil, err + } + + if err := cmd.Start(); err != nil { + return nil, nil, err + } + + o, err := io.ReadAll(stdout) + if err != nil { + return nil, nil, err + } + + e, err := io.ReadAll(stderr) + if err != nil { + return nil, nil, err + } + + if err := cmd.Wait(); err != nil { + return o, e, err + } + + return o, e, nil +} diff --git a/bin/nbconvert/ojs.go b/bin/nbconvert/ojs.go new file mode 100644 index 0000000..c28063b --- /dev/null +++ b/bin/nbconvert/ojs.go @@ -0,0 +1,27 @@ +package main + +type ObservableNotebook struct { + path string + assets []*Asset +} + +func (nb *ObservableNotebook) Path() string { + return nb.path +} + +func (nb *ObservableNotebook) Assets() []*Asset { + return nb.assets +} + +func (nb *ObservableNotebook) AddAsset(asset *Asset) []*Asset { + nb.assets = append(nb.assets, asset) + return nb.assets +} + +func (nb *ObservableNotebook) Render() error { + return Quarto(nb) +} + +func (nb *ObservableNotebook) IsIgnoredAsset(path string) bool { + return false +} diff --git a/bin/nbconvert/python.go b/bin/nbconvert/python.go new file mode 100644 index 0000000..4461c18 --- /dev/null +++ b/bin/nbconvert/python.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" +) + +func PipenvExe() string { + pipenv, err := exec.LookPath("pipenv") + if err != nil { + log.Fatalf("Could not find `pipenv`; perhaps you need to install it?") + } + return pipenv +} + +type PythonNotebook struct { + path string + assets []*Asset +} + +func (nb *PythonNotebook) Path() string { + return nb.path +} + +func (nb *PythonNotebook) Assets() []*Asset { + return nb.assets +} + +func (nb *PythonNotebook) AddAsset(asset *Asset) []*Asset { + nb.assets = append(nb.assets, asset) + return nb.assets +} + +func (nb *PythonNotebook) Instantiate() (bool, error) { + var cmd *exec.Cmd = nil + + if info, err := os.Stat(AssetPath(nb, "Pipfile")); err == nil && info.Mode().IsRegular() { + cmd = exec.Command(PipenvExe(), "install") + } else if info, err := os.Stat(AssetPath(nb, "requirements.txt")); err == nil && info.Mode().IsRegular() { + cmd = exec.Command(PipenvExe(), "install", "-r", "requirements.txt") + } + + if cmd != nil { + _, _, err := RunCommand(nb, cmd) + if err != nil { + return false, fmt.Errorf("failed to instantiate project %q: %v\n", nb.Path(), err) + } + return true, nil + } + + return false, nil +} + +func (nb *PythonNotebook) Pipenv() error { + cmd := exec.Command(PipenvExe(), "--bare", "run", QuartoExe(), "render", BaseName(nb), "--to=hugo", "--output", markdown_name) + + _, _, err := RunCommand(nb, cmd) + + return err +} + +func (nb *PythonNotebook) Render() error { + if isPipenv, err := nb.Instantiate(); err != nil { + return err + } else if isPipenv { + return nb.Pipenv() + } else { + return Quarto(nb) + } +} + +func (nb *PythonNotebook) IsIgnoredAsset(path string) bool { + return false +} diff --git a/bin/nbconvert/quarto.go b/bin/nbconvert/quarto.go new file mode 100644 index 0000000..880545f --- /dev/null +++ b/bin/nbconvert/quarto.go @@ -0,0 +1,108 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os/exec" + "path/filepath" + "regexp" +) + +func IsQuarto(path string) bool { + return filepath.Ext(path) == ".qmd" +} + +func QuartoExe() string { + quarto, err := exec.LookPath("quarto") + if err != nil { + log.Fatalf("Could not find `quarto`; perhaps you need to install it?") + } + return quarto +} + +func Quarto(nb Notebook) error { + cmd := exec.Command(QuartoExe(), "render", BaseName(nb), "--to=hugo", "--output", markdown_name) + + _, _, err := RunCommand(nb, cmd) + + return err +} + +func InferQuartoLanguage(path string) (Language, error) { + content, err := ioutil.ReadFile(path) + if err != nil { + return UnknownLang, err + } + + lang, err := InferQuartoLanguageFromHeader(content) + if err != nil { + return lang, err + } + + return InferQuartoLanguageFromBody(content) + if err != nil { + return lang, err + } + + return ObservableLang, nil +} + +func InferQuartoLanguageFromHeader(content []byte) (Language, error) { + kernelpattern := regexp.MustCompile("jupyter: (.*)") + match := kernelpattern.FindSubmatch(content) + if match != nil { + return Language(bytes.TrimSpace(match[1])), nil + } + return UnknownLang, nil +} + +func InferQuartoLanguageFromBody(content []byte) (Language, error) { + langs := map[Language]bool{} + langsWithEnv := 0 + + kernelpattern := regexp.MustCompile("```{(.*)}") + matches := kernelpattern.FindAllSubmatch(content, -1) + if matches != nil { + for _, match := range matches { + lang := LangFromQuarto(match[1]) + if _, ok := langs[lang]; !ok && lang != ObservableLang && lang != UnknownLang { + langsWithEnv += 1 + } + langs[lang] = true + } + } + + if langsWithEnv > 1 { + return UnknownLang, fmt.Errorf("mixed languages in notebook") + } + + if _, ok := langs[PythonLang]; ok { + return PythonLang, nil + } else if _, ok := langs[JuliaLang]; ok { + return JuliaLang, nil + } else if _, ok := langs[RLang]; ok { + return RLang, nil + } else if _, ok := langs[ObservableLang]; ok { + return ObservableLang, nil + } + + return UnknownLang, nil +} + +func LangFromQuarto(blang []byte) Language { + lang := string(bytes.ToLower(bytes.TrimSpace(blang))) + switch lang { + case "python": + return PythonLang + case "julia": + return JuliaLang + case "r": + return RLang + case "ojs": + return ObservableLang + default: + return UnknownLang + } +} diff --git a/bin/nbconvert/r.go b/bin/nbconvert/r.go new file mode 100644 index 0000000..d14263e --- /dev/null +++ b/bin/nbconvert/r.go @@ -0,0 +1,27 @@ +package main + +type RNotebook struct { + path string + assets []*Asset +} + +func (nb *RNotebook) Path() string { + return nb.path +} + +func (nb *RNotebook) Assets() []*Asset { + return nb.assets +} + +func (nb *RNotebook) AddAsset(asset *Asset) []*Asset { + nb.assets = append(nb.assets, asset) + return nb.assets +} + +func (nb *RNotebook) Render() error { + return Quarto(nb) +} + +func (nb *RNotebook) IsIgnoredAsset(path string) bool { + return false +} diff --git a/bin/nbconvert/util.go b/bin/nbconvert/util.go new file mode 100644 index 0000000..309e8c4 --- /dev/null +++ b/bin/nbconvert/util.go @@ -0,0 +1,66 @@ +package main + +import ( + "archive/zip" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" +) + +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return out.Close() +} + +func CopyDirectory(src, dst string) error { + cmd := exec.Command("cp", "--recursive", src, dst) + return cmd.Run() +} + +func AddFiles(w *zip.Writer, basePath, baseInZip string) error { + files, err := ioutil.ReadDir(basePath) + if err != nil { + return err + } + + for _, file := range files { + if !file.IsDir() { + data, err := ioutil.ReadFile(filepath.Join(basePath, file.Name())) + if err != nil { + return err + } + + f, err := w.Create(filepath.Join(baseInZip, file.Name())) + if err != nil { + return err + } + + _, err = f.Write(data) + if err != nil { + return err + } + } else if file.IsDir() && file.Name() != checkpoints { + newBase := filepath.Join(basePath, file.Name()) + AddFiles(w, newBase, filepath.Join(baseInZip, file.Name())) + } + } + + return nil +} diff --git a/layouts/notebooks/single.html b/layouts/notebooks/single.html index 2b9ef28..5eb844a 100644 --- a/layouts/notebooks/single.html +++ b/layouts/notebooks/single.html @@ -8,38 +8,61 @@

{{ .Title }}

-
Hide Code
-
{{ .Content }}