promptflow

doc_generation.ps1
261 строка · 11.4 Кб
Перенос по словам
1
<#
2
.DESCRIPTION
3
Script to build doc site
4

5
.EXAMPLE
6
PS> ./doc_generation.ps1 -SkipInstall # skip pip install
7
PS> ./doc_generation.ps1 -BuildLinkCheck -WarningAsError:$true -SkipInstall
8

9
#>
10
[CmdletBinding()]
11
param(
12
    [switch]$SkipInstall,
13
    [switch]$WarningAsError = $false,
14
    [switch]$BuildLinkCheck = $false,
15
    [switch]$WithReferenceDoc = $false
16
)
17

18
[string] $ScriptPath = $PSCommandPath | Split-Path -Parent
19
[string] $RepoRootPath = $ScriptPath | Split-Path -Parent | Split-Path -Parent
20
[string] $DocPath = [System.IO.Path]::Combine($RepoRootPath, "docs")
21
[string] $TempDocPath = New-TemporaryFile | % { Remove-Item $_; New-Item -ItemType Directory -Path $_ }
22
[string] $PkgSrcPath = [System.IO.Path]::Combine($RepoRootPath, "src")
23
[string] $OutPath = [System.IO.Path]::Combine($ScriptPath, "_build")
24
[string] $SphinxApiDoc = [System.IO.Path]::Combine($DocPath, "sphinx_apidoc.log")
25
[string] $SphinxBuildDoc = [System.IO.Path]::Combine($DocPath, "sphinx_build.log")
26
[string] $WarningErrorPattern = "WARNING:|ERROR:|CRITICAL:| broken "
27
[System.Collections.ArrayList]$IncludeList = @("promptflow-tracing", "promptflow-core", "promptflow-devkit", "promptflow-azure", "promptflow-rag", "promptflow-evals")
28
$apidocWarningsAndErrors = $null
29
$buildWarningsAndErrors = $null
30

31
if (-not $SkipInstall){
32
    # Prepare doc generation packages
33
    pip install pydata-sphinx-theme==0.11.0
34
    pip install sphinx==5.1
35
    pip install sphinx-copybutton==0.5.0
36
    pip install sphinx_design==0.3.0
37
    pip install sphinx-sitemap==2.2.0
38
    pip install sphinx-togglebutton==0.3.2
39
    pip install sphinxext-rediraffe==0.2.7
40
    pip install sphinxcontrib-mermaid==0.8.1
41
    pip install ipython-genutils==0.2.0
42
    pip install myst-nb==0.17.1
43
    pip install numpydoc==1.5.0
44
    pip install myst-parser==0.18.1
45
    pip install matplotlib==3.4.3
46
    pip install jinja2==3.0.1
47
    pip install jupyter-sphinx==0.4.0
48
    Write-Host "===============Finished install requirements==============="
49
}
50

51

52
function ProcessFiles {
53
    # Exclude files not mean to be in doc site
54
    $exclude_files = "README.md", "dev"
55
    foreach ($f in $exclude_files)
56
    {
57
        $full_path = [System.IO.Path]::Combine($TempDocPath, $f)
58
        Remove-Item -Path $full_path -Recurse
59
    }
60
}
61

62
Write-Host "===============PreProcess Files==============="
63
Write-Host "Copy doc to: $TempDocPath"
64
ROBOCOPY $DocPath $TempDocPath /S /NFL /NDL /XD "*.git" [System.IO.Path]::Combine($DocPath, "_scripts\_build")
65
ProcessFiles
66

67
function Update-Sub-Pkg-Index-Title {
68
    param (
69
        [string] $SubPkgRefDocPath,
70
        [string] $SubPkgName
71
    )
72
    # This is used to update the title of the promptflow.rst file in the sub package
73
    # from 'promptflow namespaces' to package name
74
    $IndexRst = [System.IO.Path]::Combine($SubPkgRefDocPath, "promptflow.rst")
75
    $IndexContent = Get-Content $IndexRst
76
    $IndexContent[0] = ("{0} package" -f $SubPkgName)
77
    $IndexContent[1] = "================================="
78
    $IndexContent[2] = ".. py:module:: promptflow"
79
    $IndexContent[3] = "   :noindex:"
80
    Set-Content $IndexRst $IndexContent
81
}
82

83
function Add-Changelog {
84
    $ChangelogFolder = [System.IO.Path]::Combine($TempDocPath, "reference\changelog")
85
    New-Item -ItemType Directory -Path $ChangelogFolder -Force
86
    Write-Host "===============Collect Package ChangeLog==============="
87
    $TocTreeContent = @("", "``````{toctree}", ":maxdepth: 1", ":hidden:", "")
88
    foreach($Item in Get-Childitem -path $PkgSrcPath)
89
    {
90
        if((-not ($IncludeList -contains $Item.Name)) -and ($Item.Name -ne "promptflow")){
91
            continue
92
        }
93
        # Collect CHANGELOG, name with package.md
94
        $ChangelogPath = [System.IO.Path]::Combine($Item.FullName, "CHANGELOG.md")
95
        $TargetChangelogPath = [System.IO.Path]::Combine($ChangelogFolder, "{0}.md" -f $Item.Name)
96
        if($Item.Name -ne "promptflow"){
97
            $TocTreeContent += $Item.name
98
        }
99
        Copy-Item -Path $ChangelogPath -Destination $TargetChangelogPath
100
    }
101
    $TocTreeContent += "``````"
102
    # Add subpackage index to promptflow changelog
103
    $PromptflowChangelog = [System.IO.Path]::Combine($ChangelogFolder, "promptflow.md")
104
    $PromptflowChangelogContent = Get-Content $PromptflowChangelog
105
    $PromptflowChangelogContent[0] = "# promptflow package"
106
    $PromptflowChangelogContent += $TocTreeContent
107
    Set-Content $PromptflowChangelog $PromptflowChangelogContent
108
}
109

110
function Add-Api-Reference {
111
    $RefDocRelativePath = "reference\python-library-reference"
112
    $RefDocPath = [System.IO.Path]::Combine($TempDocPath, $RefDocRelativePath)
113
    $PlaceHolderFile = [System.IO.Path]::Combine($RefDocPath, "promptflow.md")
114
    if(!(Test-Path $RefDocPath)){
115
        throw "Reference doc path not found. Please make sure '$RefDocRelativePath' is under '$DocPath'"
116
    }
117
    Remove-Item $PlaceHolderFile -Force
118
    $ApidocWarningsAndErrors = [System.Collections.ArrayList]::new()
119
    foreach($Item in Get-Childitem -path $PkgSrcPath){
120
        if(-not ($IncludeList -contains $Item.Name)){
121
            continue
122
        }
123
        # Build API reference doc
124
        $SubPkgPath = [System.IO.Path]::Combine($Item.FullName, "promptflow")
125
        $SubPkgRefDocPath = [System.IO.Path]::Combine($RefDocPath, $Item.Name)
126
        Write-Host "===============Build $Item Reference Doc==============="
127
        $TemplatePath = [System.IO.Path]::Combine($RepoRootPath, "scripts\docs\api_doc_templates")
128
        sphinx-apidoc --separate --module-first --no-toc --implicit-namespaces "$SubPkgPath" -o "$SubPkgRefDocPath" -t $TemplatePath | Tee-Object -FilePath $SphinxApiDoc
129
        $SubPkgWarningsAndErrors = Select-String -Path $SphinxApiDoc -Pattern $WarningErrorPattern
130
        if($SubPkgWarningsAndErrors){
131
            $ApidocWarningsAndErrors.AddRange($SubPkgWarningsAndErrors)
132
        }
133
        Update-Sub-Pkg-Index-Title $SubPkgRefDocPath $Item.Name
134
    }
135
}
136

137
function Add-Metadata{
138
    param (
139
        [string] $NotebookPath,
140
        [string] $NotebookRepoPath,
141
        [System.Collections.ArrayList] $AuthorList
142
    )
143
    if (-not $AuthorList){
144
        # Skip insert if author list not set
145
        throw "Skip Add Metadata: $NotebookPath - Author list not set"
146
        return
147
    }
148
    $NotebookContent = Get-Content $NotebookPath -Raw | ConvertFrom-Json
149
    # Covert to System.Collections.ArrayList to avoid 'Collection was of a fixed size' error.
150
    $NotebookContent.cells = [System.Collections.ArrayList]::new($NotebookContent.cells)
151
    if($NotebookContent.cells[0].source.Length -gt 1){
152
        # If the first cell length > 1, indicate there are more things than title it self in the first cell
153
        throw "Skip Add Metadata: $NotebookPath - First cell length > 1, only leave title to that cell."
154
        return
155
    }
156
    $MetadataFormat = "Authored by:&nbsp;{0}{1}"
157
    $SingleAuthor = "&nbsp;<a href='https://github.com/{0}' target='_blank'><img src='https://github.com/{0}.png' alt='Avatar' class='avatar dark-light'></a>"
158
    $JumpLink = "<a href='{0}' target='_blank'><img decoding='async' loading='lazy' src='https://img.shields.io/badge/Open%20on%20GitHub-grey?logo=github' alt='Open on GitHub' class='img_ev3q' style='float: right;'></a>" -f $NotebookRepoPath
159
    $Authors = $AuthorList | ForEach-Object { $SingleAuthor -f $_.replace("@github.com", "") }
160
    $Metadata = $MetadataFormat -f ($Authors -join ""), $JumpLink
161
    # Insert metadata to cells
162
    $MetadataCell = @{
163
        "cell_type" = "markdown";
164
        "metadata" = @{};
165
        "source" = @($Metadata)
166
    }
167
    $NotebookContent.cells.Insert(1, $MetadataCell)
168
    $NotebookContent | ConvertTo-Json -Depth 100 | Set-Content $NotebookPath
169
}
170

171
function Add-Notebook
172
{
173
    Write-Host "===============Collect Package Notebooks==============="
174
    $NotebookRootPath = [System.IO.Path]::Combine($RepoRootPath, "examples")
175
    $TargetNotebookPath = [System.IO.Path]::Combine($TempDocPath, "tutorials")
176
    # Create section list
177
    $SectionNames = "Tracing", "Prompty", "Flow", "Rag"
178
    $Sections = [ordered]@{
179
        Tracing=[System.Collections.ArrayList]::new();
180
        Prompty=[System.Collections.ArrayList]::new();
181
        Flow=[System.Collections.ArrayList]::new()
182
        Rag=[System.Collections.ArrayList]::new()
183
    }
184
    foreach($Item in Get-Childitem -path $NotebookRootPath -Recurse -Filter "*.ipynb")
185
    {
186
        # Notebook to build must have metadata: {"build_doc": {"category": "local/azure"}}
187
        $NotebookContent = Get-Content $Item.FullName -Raw | ConvertFrom-Json
188
        if(-not $NotebookContent.metadata.build_doc){
189
            continue
190
        }
191
        $RepoPath = $Item.FullName.Replace($RepoRootPath, "https://github.com/microsoft/promptflow/tree/main/")
192
        $SectionName = $NotebookContent.metadata.build_doc.section
193
        [int]$Weight = $NotebookContent.metadata.build_doc.weight
194
        $Category = $NotebookContent.metadata.build_doc.category
195
        $AuthorList = $NotebookContent.metadata.build_doc.author
196
        # If category is 'azure', add 1000 to weight
197
        if($Category -eq "azure"){
198
            $Weight += 1000
199
        }
200
        # Add ItemName, Category tuple to sections
201
        $Sections[$SectionName].Add([Tuple]::Create($Item.Name.Replace(".ipynb", ""), $Weight))
202
        # Copy notebook to doc path
203
        Write-Host "Adding Notebook $Item ..."
204
        $MediaDir = $Item.FullName + '\..\media'
205
        Copy-Item -Path $Item.FullName -Destination $TargetNotebookPath
206
        if(Test-Path $MediaDir){
207
            # copy image referenced in notebook
208
            Write-Host "Copying media files from $MediaDir ..."
209
            Copy-Item -Path $MediaDir -Destination $TargetNotebookPath -Recurse -Force
210
        }
211
        # Append metadata to notebook
212
        $CopiedNotebookPath = [System.IO.Path]::Combine($TargetNotebookPath, $Item.Name)
213
        Add-Metadata $CopiedNotebookPath $RepoPath $AuthorList
214
    }
215
    # Reverse sort each section list by Weight
216
    foreach($SectionName in $SectionNames){
217
        $Sections[$SectionName] = $Sections[$SectionName] | Sort-Object -Property { $_.Item2 }
218
    }
219
    $TocTreeContent = @("", "``````{{toctree}}", ":caption: {0}", ":hidden:", ":maxdepth: 1", "", "{1}", "``````")
220
    # Build toctree content for each section, append to tutorials index.md
221
    $TutorialIndex = [System.IO.Path]::Combine($TargetNotebookPath, "index.md")
222
    foreach($SectionName in $SectionNames){
223
        $SectionTocTree = $TocTreeContent -join "`n"
224
        # Join Item1 to a string in list
225
        $ExampleList = ($Sections[$SectionName] | ForEach-Object { $_.Item1 }) -join "`n"
226
        $SectionTocTree = $SectionTocTree -f $SectionName, $ExampleList
227
        Write-Debug $SectionTocTree
228
        Add-Content -Path $TutorialIndex -Value $SectionTocTree
229
    }
230
}
231

232
if($WithReferenceDoc){
233
    Add-Api-Reference
234
}
235
# Build subpackage changelog
236
Add-Changelog
237
# Build notebook examples
238
Add-Notebook
239

240
Write-Host "===============Build Documentation with internal=${Internal}==============="
241
$BuildParams = [System.Collections.ArrayList]::new()
242
if($WarningAsError){
243
    $BuildParams.Add("-W")
244
    $BuildParams.Add("--keep-going")
245
}
246
if($BuildLinkCheck){
247
    $BuildParams.Add("-blinkcheck")
248
}
249
sphinx-build $TempDocPath $OutPath -c $ScriptPath $BuildParams -v | Tee-Object -FilePath $SphinxBuildDoc
250
$buildWarningsAndErrors = Select-String -Path $SphinxBuildDoc -Pattern $WarningErrorPattern
251

252
Write-Host "Clean path: $TempDocPath"
253
Remove-Item $TempDocPath -Recurse -Confirm:$False -Force
254

255

256
if ($buildWarningsAndErrors) {
257
    Write-Host "=============== Build warnings and errors ==============="
258
    foreach ($line in $buildWarningsAndErrors) {
259
        Write-Host $line -ForegroundColor Red
260
    }
261
}
promptflow

Использование cookies
