tokenizers

1
// These three variables below need to be updated at each release for the selectors.
2

3
const languages = [ "rust", "python", "node" ];
4
// Last stable version for each language
5
const stableVersion = {
6
    "rust": "master",
7
    "python": "v0.10.0",
8
    "node": "master"
9
}
10
// Dictionary doc folder to Label for each language
11
const versionMapping = {
12
    "rust": {
13
        "master": "master",
14
    },
15
    "python": {
16
        "master": "master",
17
        "v0.9.4": "v0.9.4",
18
        "v0.10.0": "v0.10.0",
19
    },
20
    "node": {
21
        "master": "master",
22
    }
23
};
24

25
// Dictionnary language name to Label
26
const languageName = {
27
    "rust": "Rust",
28
    "python": "Python",
29
    "node": "Node.js"
30
};
31
const defaultLanguage = "python";
32

33
function addIcon() {
34
    const huggingFaceLogo =
35
        "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg";
36
    const image = document.createElement("img");
37
    image.setAttribute("src", huggingFaceLogo);
38

39
    const div = document.createElement("div");
40
    div.appendChild(image);
41
    div.style.textAlign = 'center';
42
    div.style.paddingTop = '30px';
43
    div.style.backgroundColor = '#6670FF';
44

45
    const scrollDiv = document.querySelector(".wy-side-scroll");
46
    scrollDiv.prepend(div);
47
}
48

49
function addCustomFooter() {
50
    const customFooter = document.createElement("div");
51
    const questionOrIssue = document.createElement("div");
52
    questionOrIssue.innerHTML =
53
        "Stuck? Read our <a href='https://medium.com/huggingface'>Blog posts</a>" +
54
        " or <a href='https://github.com/huggingface/tokenizers'>Create an issue</a>";
55
    customFooter.appendChild(questionOrIssue);
56
    customFooter.classList.add("footer");
57

58
    const social = document.createElement("div");
59
    social.classList.add("footer__Social");
60

61
    const imageDetails = [ {
62
        link: "https://huggingface.co",
63
        imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg"
64
    }, {
65
        link: "https://twitter.com/huggingface",
66
        imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg"
67
    }, {
68
        link: "https://github.com/huggingface",
69
        imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg"
70
    }, {
71
        link: "https://www.linkedin.com/company/huggingface/",
72
        imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg"
73
    } ];
74

75
    imageDetails.forEach(imageLinks => {
76
        const link = document.createElement("a");
77
        const image = document.createElement("img");
78
        image.src = imageLinks.imageLink;
79
        link.href = imageLinks.link;
80
        image.style.width = "30px";
81
        image.classList.add("footer__CustomImage");
82
        link.appendChild(image);
83
        social.appendChild(link);
84
    });
85

86
    customFooter.appendChild(social);
87
    document.querySelector("footer").appendChild(customFooter);
88
}
89

90
function addGithubButton() {
91
    const div = `
92
        <div class="github-repo">
93
            <a class="github-button"
94
               href="https://github.com/huggingface/tokenizers"
95
               data-size="large"
96
               data-show-count="true"
97
               aria-label="Star huggingface/tokenizers on GitHub">
98
                Star
99
            </a>
100
        </div>
101
    `;
102
    document.querySelector(".wy-side-nav-search .icon-home").insertAdjacentHTML('afterend', div);
103
}
104

105
function addVersionControl() {
106
    // Default language and version
107
    let language = defaultLanguage;
108

109
    // To grab the version currently in view, we parse the url
110
    let parts = location.pathname.split('/');
111
    const languageIndex = parts.findIndex((part) => languages.includes(part));
112
    language = parts[languageIndex];
113
    let version  = stableVersion[language];
114

115
    const versionIndex  = languageIndex + 1;
116
    // If a version is specified, update it
117
    if (parts[versionIndex] != "" && !parts[versionIndex].endsWith(".html")) {
118
        // If `latest`, let's keep the default (should be the explicit latest version)
119
        if (parts[versionIndex] != "latest") {
120
            version = parts[versionIndex];
121
        }
122
    // Otherwise redirect to the latest (if not opening locally)
123
    } else if (!parts[parts.length - 1].endsWith(".html")) {
124
        return window.location.pathname = [language, version, parts.splice(versionIndex)].join("/");
125
    // Opening locally, just don't show the version/language selector
126
    } else {
127
        return
128
    }
129

130
    // Language Menu
131
    const languageMenu = document.createElement("div");
132
    languageMenu.classList.add("menu-dropdown");
133
    languageMenu.innerHTML = languages.map((lang) => {
134
        let isVersion = false;
135
        let updatedParts = parts.map((l, i) => {
136
            if (isVersion) {
137
                isVersion = false;
138
                return 'latest';
139
            }
140
            if (i == languageIndex) {
141
                isVersion = true;
142
                return lang;
143
            } else {
144
                return l;
145
            }
146
        });
147
        return `
148
            <a class="dropdown-link ${lang == language? 'active' : ''}"
149
                href=${updatedParts.join("/")}>
150
                ${languageName[lang]}
151
            </a>
152
        `;
153
    }).join("\n");
154

155
    // Version Menu
156
    const versionMenu = document.createElement("div");
157
    versionMenu.classList.add("menu-dropdown");
158
    versionMenu.innerHTML = Object.entries(versionMapping[language]).map(([key, value]) => {
159
        let updatedParts = parts.map((v, i) => {
160
            if (i == versionIndex) {
161
                return key;
162
            } else {
163
                return v
164
            }
165
        });
166
        return `
167
            <a class="dropdown-link ${key == version ? 'active' : ''}"
168
                href="${updatedParts.join('/')}">
169
                    ${value}
170
            </a>
171
        `;
172
    }).join("\n");
173

174
    // Language button
175
    const languageButton = document.createElement("div");
176
    languageButton.classList.add("dropdown-button");
177
    languageButton.innerText = languageName[language].concat(" ▼");
178

179
    languageButton.addEventListener("click", () => {
180
        versionMenu.classList.remove("show");
181
        languageMenu.classList.toggle("show");
182
        languageButton.classList.toggle("active");
183
    });
184

185
    // Button for version selection
186
    const versionButton = document.createElement("div");
187
    versionButton.classList.add("dropdown-button");
188
    versionButton.innerText = version.concat(" ▼");
189

190
    // Toggle the menu when we click on the button
191
    versionButton.addEventListener("click", () => {
192
        languageMenu.classList.remove("show");
193
        versionMenu.classList.toggle("show");
194
        versionButton.classList.toggle("active");
195
    });
196

197
    // Hide the menu when we click elsewhere
198
    window.addEventListener("click", (event) => {
199
        if (event.target != languageButton){
200
            languageButton.classList.remove('active');
201
            languageMenu.classList.remove('show');
202
        }
203
        if (event.target != versionButton){
204
            versionButton.classList.remove('active');
205
            versionMenu.classList.remove('show');
206
        }
207
    });
208

209
    const buttonContainer = document.createElement("div");
210
    buttonContainer.classList.add("button-container");
211
    buttonContainer.appendChild(languageButton);
212
    buttonContainer.appendChild(versionButton);
213

214
    // Container
215
    const div = document.createElement("div");
216
    div.classList.add("selectors");
217
    div.appendChild(buttonContainer);
218
    div.appendChild(languageMenu);
219
    div.appendChild(versionMenu);
220
    div.style.paddingTop = '25px';
221
    div.style.backgroundColor = '#6670FF';
222
    div.style.display = 'block';
223
    div.style.textAlign = 'center';
224

225
    const scrollDiv = document.querySelector(".wy-side-scroll");
226
    scrollDiv.insertBefore(div, scrollDiv.children[1]);
227
}
228

229
function addHfMenu() {
230
    const div = `
231
    <div class="hf-menu">
232
        <a href="/welcome">🔥 Sign in</a>
233
        <a href="/models">🚀 Models</a>
234
        <a href="http://discuss.huggingface.co">💬 Forum</a>
235
    </div>
236
    `;
237
    document.body.insertAdjacentHTML('afterbegin', div);
238
}
239

240
/*!
241
 * github-buttons v2.2.10
242
 * (c) 2019 なつき
243
 * @license BSD-2-Clause
244
 */
245
/**
246
 * modified to run programmatically
247
 */
248
function parseGithubButtons (){"use strict";var e=window.document,t=e.location,o=window.encodeURIComponent,r=window.decodeURIComponent,n=window.Math,a=window.HTMLElement,i=window.XMLHttpRequest,l="https://unpkg.com/github-buttons@2.2.10/dist/buttons.html",c=i&&i.prototype&&"withCredentials"in i.prototype,d=c&&a&&a.prototype.attachShadow&&!a.prototype.attachShadow.prototype,s=function(e,t,o){e.addEventListener?e.addEventListener(t,o):e.attachEvent("on"+t,o)},u=function(e,t,o){e.removeEventListener?e.removeEventListener(t,o):e.detachEvent("on"+t,o)},h=function(e,t,o){var r=function(n){return u(e,t,r),o(n)};s(e,t,r)},f=function(e,t,o){var r=function(n){if(t.test(e.readyState))return u(e,"readystatechange",r),o(n)};s(e,"readystatechange",r)},p=function(e){return function(t,o,r){var n=e.createElement(t);if(o)for(var a in o){var i=o[a];null!=i&&(null!=n[a]?n[a]=i:n.setAttribute(a,i))}if(r)for(var l=0,c=r.length;l<c;l++){var d=r[l];n.appendChild("string"==typeof d?e.createTextNode(d):d)}return n}},g=p(e),b=function(e){var t;return function(){t||(t=1,e.apply(this,arguments))}},m="body{margin:0}a{color:#24292e;text-decoration:none;outline:0}.octicon{display:inline-block;vertical-align:text-top;fill:currentColor}.widget{ display:inline-block;overflow:hidden;font-family:-apple-system, BlinkMacSystemFont, \"Segoe UI\", Helvetica, Arial, sans-serif;font-size:0;white-space:nowrap;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.btn,.social-count{display:inline-block;height:14px;padding:2px 5px;font-size:11px;font-weight:600;line-height:14px;vertical-align:bottom;cursor:pointer;border:1px solid #c5c9cc;border-radius:0.25em}.btn{background-color:#eff3f6;background-image:-webkit-linear-gradient(top, #fafbfc, #eff3f6 90%);background-image:-moz-linear-gradient(top, #fafbfc, #eff3f6 90%);background-image:linear-gradient(180deg, #fafbfc, #eff3f6 90%);background-position:-1px -1px;background-repeat:repeat-x;background-size:110% 110%;border-color:rgba(27,31,35,0.2);-ms-filter:\"progid:DXImageTransform.Microsoft.Gradient(startColorstr='#FFFAFBFC', endColorstr='#FFEEF2F5')\";*filter:progid:DXImageTransform.Microsoft.Gradient(startColorstr='#FFFAFBFC', endColorstr='#FFEEF2F5')}.btn:active{background-color:#e9ecef;background-image:none;border-color:#a5a9ac;border-color:rgba(27,31,35,0.35);box-shadow:inset 0 0.15em 0.3em rgba(27,31,35,0.15)}.btn:focus,.btn:hover{background-color:#e6ebf1;background-image:-webkit-linear-gradient(top, #f0f3f6, #e6ebf1 90%);background-image:-moz-linear-gradient(top, #f0f3f6, #e6ebf1 90%);background-image:linear-gradient(180deg, #f0f3f6, #e6ebf1 90%);border-color:#a5a9ac;border-color:rgba(27,31,35,0.35);-ms-filter:\"progid:DXImageTransform.Microsoft.Gradient(startColorstr='#FFF0F3F6', endColorstr='#FFE5EAF0')\";*filter:progid:DXImageTransform.Microsoft.Gradient(startColorstr='#FFF0F3F6', endColorstr='#FFE5EAF0')}.social-count{position:relative;margin-left:5px;background-color:#fff}.social-count:focus,.social-count:hover{color:#0366d6}.social-count b,.social-count i{position:absolute;top:50%;left:0;display:block;width:0;height:0;margin:-4px 0 0 -4px;border:solid transparent;border-width:4px 4px 4px 0;_line-height:0;_border-top-color:red !important;_border-bottom-color:red !important;_border-left-color:red !important;_filter:chroma(color=red)}.social-count b{border-right-color:#c5c9cc}.social-count i{margin-left:-3px;border-right-color:#fff}.lg .btn,.lg .social-count{height:16px;padding:5px 10px;font-size:12px;line-height:16px}.lg .social-count{margin-left:6px}.lg .social-count b,.lg .social-count i{margin:-5px 0 0 -5px;border-width:5px 5px 5px 0}.lg .social-count i{margin-left:-4px}\n",v={"mark-github":{width:16,height:16,path:'<path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"/>'},eye:{width:16,height:16,path:'<path fill-rule="evenodd" d="M8.06 2C3 2 0 8 0 8s3 6 8.06 6C13 14 16 8 16 8s-3-6-7.94-6zM8 12c-2.2 0-4-1.78-4-4 0-2.2 1.8-4 4-4 2.22 0 4 1.8 4 4 0 2.22-1.78 4-4 4zm2-4c0 1.11-.89 2-2 2-1.11 0-2-.89-2-2 0-1.11.89-2 2-2 1.11 0 2 .89 2 2z"/>'},star:{width:14,height:16,path:'<path fill-rule="evenodd" d="M14 6l-4.9-.64L7 1 4.9 5.36 0 6l3.6 3.26L2.67 14 7 11.67 11.33 14l-.93-4.74L14 6z"/>'},"repo-forked":{width:10,height:16,path:'<path fill-rule="evenodd" d="M8 1a1.993 1.993 0 0 0-1 3.72V6L5 8 3 6V4.72A1.993 1.993 0 0 0 2 1a1.993 1.993 0 0 0-1 3.72V6.5l3 3v1.78A1.993 1.993 0 0 0 5 15a1.993 1.993 0 0 0 1-3.72V9.5l3-3V4.72A1.993 1.993 0 0 0 8 1zM2 4.2C1.34 4.2.8 3.65.8 3c0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3 10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2zm3-10c-.66 0-1.2-.55-1.2-1.2 0-.65.55-1.2 1.2-1.2.65 0 1.2.55 1.2 1.2 0 .65-.55 1.2-1.2 1.2z"/>'},"issue-opened":{width:14,height:16,path:'<path fill-rule="evenodd" d="M7 2.3c3.14 0 5.7 2.56 5.7 5.7s-2.56 5.7-5.7 5.7A5.71 5.71 0 0 1 1.3 8c0-3.14 2.56-5.7 5.7-5.7zM7 1C3.14 1 0 4.14 0 8s3.14 7 7 7 7-3.14 7-7-3.14-7-7-7zm1 3H6v5h2V4zm0 6H6v2h2v-2z"/>'},"cloud-download":{width:16,height:16,path:'<path fill-rule="evenodd" d="M9 12h2l-3 3-3-3h2V7h2v5zm3-8c0-.44-.91-3-4.5-3C5.08 1 3 2.92 3 5 1.02 5 0 6.52 0 8c0 1.53 1 3 3 3h3V9.7H3C1.38 9.7 1.3 8.28 1.3 8c0-.17.05-1.7 1.7-1.7h1.3V5c0-1.39 1.56-2.7 3.2-2.7 2.55 0 3.13 1.55 3.2 1.8v1.2H12c.81 0 2.7.22 2.7 2.2 0 2.09-2.25 2.2-2.7 2.2h-2V11h2c2.08 0 4-1.16 4-3.5C16 5.06 14.08 4 12 4z"/>'}},w={},x=function(e,t,o){var r=p(e.ownerDocument),n=e.appendChild(r("style",{type:"text/css"}));n.styleSheet?n.styleSheet.cssText=m:n.appendChild(e.ownerDocument.createTextNode(m));var a,l,d=r("a",{className:"btn",href:t.href,target:"_blank",innerHTML:(a=t["data-icon"],l=/^large$/i.test(t["data-size"])?16:14,a=(""+a).toLowerCase().replace(/^octicon-/,""),{}.hasOwnProperty.call(v,a)||(a="mark-github"),'<svg version="1.1" width="'+l*v[a].width/v[a].height+'" height="'+l+'" viewBox="0 0 '+v[a].width+" "+v[a].height+'" class="octicon octicon-'+a+'" aria-hidden="true">'+v[a].path+"</svg>"),"aria-label":t["aria-label"]||void 0},[" ",r("span",{},[t["data-text"]||""])]);/\.github\.com$/.test("."+d.hostname)?/^https?:\/\/((gist\.)?github\.com\/[^\/?#]+\/[^\/?#]+\/archive\/|github\.com\/[^\/?#]+\/[^\/?#]+\/releases\/download\/|codeload\.github\.com\/)/.test(d.href)&&(d.target="_top"):(d.href="#",d.target="_self");var u,h,g,x,y=e.appendChild(r("div",{className:"widget"+(/^large$/i.test(t["data-size"])?" lg":"")},[d]));/^(true|1)$/i.test(t["data-show-count"])&&"github.com"===d.hostname&&(u=d.pathname.replace(/^(?!\/)/,"/").match(/^\/([^\/?#]+)(?:\/([^\/?#]+)(?:\/(?:(subscription)|(fork)|(issues)|([^\/?#]+)))?)?(?:[\/?#]|$)/))&&!u[6]?(u[2]?(h="/repos/"+u[1]+"/"+u[2],u[3]?(x="subscribers_count",g="watchers"):u[4]?(x="forks_count",g="network"):u[5]?(x="open_issues_count",g="issues"):(x="stargazers_count",g="stargazers")):(h="/users/"+u[1],g=x="followers"),function(e,t){var o=w[e]||(w[e]=[]);if(!(o.push(t)>1)){var r=b(function(){for(delete w[e];t=o.shift();)t.apply(null,arguments)});if(c){var n=new i;s(n,"abort",r),s(n,"error",r),s(n,"load",function(){var e;try{e=JSON.parse(n.responseText)}catch(e){return void r(e)}r(200!==n.status,e)}),n.open("GET",e),n.send()}else{var a=this||window;a._=function(e){a._=null,r(200!==e.meta.status,e.data)};var l=p(a.document)("script",{async:!0,src:e+(/\?/.test(e)?"&":"?")+"callback=_"}),d=function(){a._&&a._({meta:{}})};s(l,"load",d),s(l,"error",d),l.readyState&&f(l,/de|m/,d),a.document.getElementsByTagName("head")[0].appendChild(l)}}}.call(this,"https://api.github.com"+h,function(e,t){if(!e){var n=t[x];y.appendChild(r("a",{className:"social-count",href:t.html_url+"/"+g,target:"_blank","aria-label":n+" "+x.replace(/_count$/,"").replace("_"," ").slice(0,n<2?-1:void 0)+" on GitHub"},[r("b"),r("i"),r("span",{},[(""+n).replace(/\B(?=(\d{3})+(?!\d))/g,",")])]))}o&&o(y)})):o&&o(y)},y=window.devicePixelRatio||1,C=function(e){return(y>1?n.ceil(n.round(e*y)/y*2)/2:n.ceil(e))||0},F=function(e,t){e.style.width=t[0]+"px",e.style.height=t[1]+"px"},k=function(t,r){if(null!=t&&null!=r)if(t.getAttribute&&(t=function(e){for(var t={href:e.href,title:e.title,"aria-label":e.getAttribute("aria-label")},o=["icon","text","size","show-count"],r=0,n=o.length;r<n;r++){var a="data-"+o[r];t[a]=e.getAttribute(a)}return null==t["data-text"]&&(t["data-text"]=e.textContent||e.innerText),t}(t)),d){var a=g("span",{title:t.title||void 0});x(a.attachShadow({mode:"closed"}),t,function(){r(a)})}else{var i=g("iframe",{src:"javascript:0",title:t.title||void 0,allowtransparency:!0,scrolling:"no",frameBorder:0});F(i,[0,0]),i.style.border="none";var c=function(){var a,d=i.contentWindow;try{a=d.document.body}catch(t){return void e.body.appendChild(i.parentNode.removeChild(i))}u(i,"load",c),x.call(d,a,t,function(e){var a=function(e){var t=e.offsetWidth,o=e.offsetHeight;if(e.getBoundingClientRect){var r=e.getBoundingClientRect();t=n.max(t,C(r.width)),o=n.max(o,C(r.height))}return[t,o]}(e);i.parentNode.removeChild(i),h(i,"load",function(){F(i,a)}),i.src=l+"#"+(i.name=function(e){var t=[];for(var r in e){var n=e[r];null!=n&&t.push(o(r)+"="+o(n))}return t.join("&")}(t)),r(i)})};s(i,"load",c),e.body.appendChild(i)}};t.protocol+"//"+t.host+t.pathname===l?x(e.body,function(e){for(var t={},o=e.split("&"),n=0,a=o.length;n<a;n++){var i=o[n];if(""!==i){var l=i.split("=");t[r(l[0])]=null!=l[1]?r(l.slice(1).join("=")):void 0}}return t}(window.name||t.hash.replace(/^#/,""))):function(t){if(/m/.test(e.readyState)||!/g/.test(e.readyState)&&!e.documentElement.doScroll)setTimeout(t);else if(e.addEventListener){var o=b(t);h(e,"DOMContentLoaded",o),h(window,"load",o)}else f(e,/m/,t)}(function(){for(var t=e.querySelectorAll?e.querySelectorAll("a.github-button"):function(){for(var t=[],o=e.getElementsByTagName("a"),r=0,n=o.length;r<n;r++)~(" "+o[r].className+" ").replace(/[ \t\n\f\r]+/g," ").indexOf(" github-button ")&&t.push(o[r]);return t}(),o=0,r=t.length;o<r;o++)!function(e){k(e,function(t){e.parentNode.replaceChild(t,e)})}(t[o])})};
249

250

251
function onLoad() {
252
    addIcon();
253
    addVersionControl();
254
    addCustomFooter();
255
    addGithubButton();
256
    parseGithubButtons();
257
    addHfMenu();
258
}
259

260
window.addEventListener("load", onLoad);
261