docset-creator

Utility module used to create docsets. Supports

1. HTML page parsing
2. easy config

Installation

``
yarn add docset-generator
`

Create the direcoty

`
|- docs
|- { all files to be included. SiteSucker or another utility application can be used to download offline files }
|- src
|- options.js
`

Running the app

If you wan't a dry run, add --dry-run CLI arg.

`
docset-creator --chrome-exe-path /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome
`

note: the chome exe path is only required if the selectors attribute is provided

options.js

`
// the name of the index file e.g. "index.html"
indexFileName?: string;
// the path of the index file directory relative to the base path
indexFileDirPath?: string;
// a path prefix that should be excluded from all doc entries (in case URLs were copy/pasted from browser)
excludePathPrefix?: string;
// a path prefix that should be included for each docset entry URL
includePathPrefix?: string;
// the equivalent of the package name for the docset
docsetIdentifier: string;
// human readable name of the docset (will default to docsetIdentifier)
docsetName?: string;
// javascript is enabled by default, false if it should be disabled
isJavascriptEnabled?: boolean;
// Dash plist.info value, defaults to docsetIdentifier
docsetPlatformFamily?: string;
// Dash plist.info value for website url
fallbackUrl?: string;
// any additionl selectors to populate entries
selectors?: Selector[];
// all docset entries
entries?: Entries;
`

types

`
export interface Selector extends SubSelector {
url: string;
waitFor?: string;
}

/ handler function used to navigate to an HTML file and return associated docset entries /
export interface SelectorHandler {
(data: BrowserData): Promise;
}

export interface BrowserData {
/ return the element attribute as a promise /
attr: (key: string) => Promise;
/ return the element attribute as a promise /
innerText: () => Promise;
/ return the element outer html as a promise /
outerHTML: () => Promise;
/ return a BrowserData representing a querySelector value using the current element as the root as a promise /
single: (selector: string) => Promise;
/ return a BrowserData array representing a querySelectorAll value using the current element as the root as a promise /
all: (selector: string) => Promise;
/* using the selector value, execute the handler function with the selector value and append the items to the entries provided as the 2nd parameter
and return the aggregated results */
addTo: (selector: SubSelector, entries: Entries) => Promise;
/ navigate to a new URL /
goTo: (url: string, waitFor?: string) => Promise;
}

/ selector data provided to the addTo function of BrowserData /
export interface SubSelector {
/ the selector value /
selector: string;
/ the handler function provided called for each selector match /
value: (data: BrowserData) => Promise;
}

/ main selector object provided as the selectors options value /
export interface Selector extends SubSelector {
/ the URL to navigate to /
url: string;
/ optional element selector to wait before processing /
waitFor?: string;
}

`

Example

`
function mainSelector(options) {
return {
url: options.url,
selector: ".toctree-l1.current",
value: async (data) => {
const entry = await data.single("a");
const text = await entry.innerText();
const url = await entry.attr("href");
const name = options.name || text;
const rtn = {
Entry: {
[name]: url,
},
};
return data.addTo(
{
selector: ".toctree-l2",
value: async (data) => {
const entry = await data.single("a");
const text = await entry.innerText();
const url = await entry.attr("href");
return {
Entry: {
[name + ": " + text]: url,
},
};
},
},
rtn
);
},
};
}

function l3Selector(options) {
return {
url: options.url,
selector: ".toctree-l3",
value: async (data) => {
const entry = await data.single("a");
const url = await entry.attr("href");
let name = await entry.innerText();
if (options.namePrefix) {
name = options.namePrefix + ": " + name;
}
return {
[options.type]: {
[name]: url,
},
};
},
};
}

function generalReferenceSelector() {
return {
url: "https://docs.snowflake.com/en/sql-reference.html",
selector: ".left-sidebar .current .toctree-l2",
value: async (data) => {
const entry = await data.single("a");
const name = await entry.innerText();
if (!name || name === "Parameters") {
return;
}
const url = await entry.attr("href");
data = await data.goTo(url, ".toctree-l2");
return data.addTo({
selector: ".current .toctree-l3",
value: async (data) => {
const entry = await data.single("a");
const text = await entry.innerText();
const url = await entry.attr("href");
const _name = name + ": " + text;
const rtn = {
Section: {
[_name]: url,
},
};
return rtn;
},
});
},
};
}

const options = {
docsetIdentifier: "snowflake",
docsetName: "Snowflake",
// the index file name
indexFileName: "index.html",
// if the index file is not in the root directory
indexFileDirPath: "en",
fallbackUrl: "https://docs.snowflake.com/en/",
// all entry path values will have this path prefix removed (useful if path values are copied from URLs)
excludePathPrefix: "https://docs.snowflake.com/en/",

selectors: [
mainSelector({
url: "https://docs.snowflake.com/en/user-guide-getting-started.html",
}),
mainSelector({
name: "Introduction",
url: "https://docs.snowflake.com/en/user-guide-intro.html",
}),
mainSelector({
url: "https://docs.snowflake.com/en/release-notes.html",
}),
mainSelector({
url: "https://docs.snowflake.com/en/user-guide-connecting.html",
}),
mainSelector({
url: "https://docs.snowflake.com/en/user-guide-data-load.html",
}),
mainSelector({
url: "https://docs.snowflake.com/en/user-guide-data-unload.html",
}),
mainSelector({
name: "Sharing Data Securely",
url: "https://docs.snowflake.com/en/user-guide-data-share.html",
}),
mainSelector({
name: "Managing Your Account",
url: "https://docs.snowflake.com/en/user-guide-admin.html",
}),
mainSelector({
name: "Managing Security",
url: "https://docs.snowflake.com/en/user-guide-admin-security.html",
}),
mainSelector({
name: "Managing Security",
url: "https://docs.snowflake.com/en/user-guide-admin-security.html",
}),
l3Selector({
url: "https://docs.snowflake.com/en/sql-reference/parameters.html",
type: "Parameter",
}),
generalReferenceSelector(),
mainSelector({
name: "SQL Command Reference",
url: "https://docs.snowflake.com/en/sql-reference-commands.html",
}),
mainSelector({
name: "SQL Function Reference",
url: "https://docs.snowflake.com/en/sql-reference-functions.html",
}),
],
};
module.exports = options;
`

DocsetEntries

`
export interface DocsetEntries {
Annotation?: Record;
Attribute?: Record;
Binding?: Record;
Builtin?: Record;
Callback?: Record;
Category?: Record;
Class?: Record;
Command?: Record;
Component?: Record;
Constant?: Record;
Constructor?: Record;
Define?: Record;
Delegate?: Record;
Diagram?: Record;
Directive?: Record;
Element?: Record;
Entry?: Record;
Enum?: Record;
Environment?: Record;
Error?: Record;
Event?: Record;
Exception?: Record;
Extension?: Record;
Field?: Record;
File?: Record;
Filter?: Record;
Framework?: Record;
Function?: Record;
Global?: Record;
Guide?: Record;
Hook?: Record;
Instance?: Record;
Instruction?: Record;
Interface?: Record;
Keyword?: Record;
Library?: Record;
Literal?: Record;
Macro?: Record;
Method?: Record;
Mixin?: Record;
Modifier?: Record;
Module?: Record;
Namespace?: Record;
Notation?: Record;
Object?: Record;
Operator?: Record;
Option?: Record;
Package?: Record;
Parameter?: Record;
Plugin?: Record;
Procedure?: Record;
Property?: Record;
Protocol?: Record;
Provider?: Record;
Provisioner?: Record;
Query?: Record;
Record?: Record;
Resource?: Record;
Sample?: Record;
Section?: Record;
Service?: Record;
Setting?: Record;
Shortcut?: Record;
Statement?: Record;
Struct?: Record;
Style?: Record;
Subroutine?: Record;
Tag?: Record;
Test?: Record;
Trait?: Record;
Type?: Record;
Union?: Record;
Value?: Record;
Variable?: Record;
Word?: Record;
}
``