src/search/node/FindInFilesDomain.js - master - githubmirror/brackets

src/search/node/FindInFilesDomain.js
685 строк24 KB
ficristo
ESLint: enable strict rule
25 авг 2016, 23:24
25 авг 2016, 23:248370a08
/*
 * Copyright (c) 2015 - present Adobe Systems Incorporated. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 */

/*eslint-env node */
/*jslint node: true */
/*global setImmediate */
"use strict";

var fs = require("fs"),
    projectCache = [],
    files,
    _domainManager,
    MAX_FILE_SIZE_TO_INDEX = 16777216, //16MB
    MAX_DISPLAY_LENGTH = 200,
    MAX_TOTAL_RESULTS = 100000, // only 100,000 search results are supported
    MAX_RESULTS_IN_A_FILE = MAX_TOTAL_RESULTS,
    MAX_RESULTS_TO_RETURN = 120;

var results = {},
    numMatches = 0,
    numFiles = 0,
    evaluatedMatches,
    foundMaximum = false,
    exceedsMaximum = false,
    currentCrawlIndex = 0,
    savedSearchObject = null,
    lastSearchedIndex = 0,
    crawlComplete = false,
    crawlEventSent = false,
    collapseResults = false,
    cacheSize = 0;

/**
 * Copied from StringUtils.js
 * Returns a line number corresponding to an offset in some text. The text can
 * be specified as a single string or as an array of strings that correspond to
 * the lines of the string.
 *
 * Specify the text in lines when repeatedly calling the function on the same
 * text in a loop. Use getLines() to divide the text into lines, then repeatedly call
 * this function to compute a line number from the offset.
 *
 * @param {string | Array.<string>} textOrLines - string or array of lines from which
 *      to compute the line number from the offset
 * @param {number} offset
 * @return {number} line number
 */
function offsetToLineNum(textOrLines, offset) {
    if (Array.isArray(textOrLines)) {
        var lines = textOrLines,
            total = 0,
            line;
        for (line = 0; line < lines.length; line++) {
            if (total < offset) {
                // add 1 per line since /n were removed by splitting, but they needed to
                // contribute to the total offset count
                total += lines[line].length + 1;
            } else if (total === offset) {
                return line;
            } else {
                return line - 1;
            }
        }

        // if offset is NOT over the total then offset is in the last line
        if (offset <= total) {
            return line - 1;
        } else {
            return undefined;
        }
    } else {
        return textOrLines.substr(0, offset).split("\n").length - 1;
    }
}

/**
 * Searches through the contents and returns an array of matches
 * @param {string} contents
 * @param {RegExp} queryExpr
 * @return {!Array.<{start: {line:number,ch:number}, end: {line:number,ch:number}, line: string}>}
 */
function getSearchMatches(contents, queryExpr) {
    if (!contents) {
        return;
    }
    // Quick exit if not found or if we hit the limit
    if (foundMaximum || contents.search(queryExpr) === -1) {
        return [];
    }

    var match, lineNum, line, ch, totalMatchLength, matchedLines, numMatchedLines, lastLineLength, endCh,
        padding, leftPadding, rightPadding, highlightOffset, highlightEndCh,
        lines   = contents.split("\n"),
        matches = [];

    while ((match = queryExpr.exec(contents)) !== null) {
        lineNum          = offsetToLineNum(lines, match.index);
        line             = lines[lineNum];
        ch               = match.index - contents.lastIndexOf("\n", match.index) - 1;  // 0-based index
        matchedLines     = match[0].split("\n");
        numMatchedLines  = matchedLines.length;
        totalMatchLength = match[0].length;
        lastLineLength   = matchedLines[matchedLines.length - 1].length;
        endCh            = (numMatchedLines === 1 ? ch + totalMatchLength : lastLineLength);
        highlightEndCh   = (numMatchedLines === 1 ? endCh : line.length);
        highlightOffset  = 0;

        if (highlightEndCh <= MAX_DISPLAY_LENGTH) {
            // Don't store more than 200 chars per line
            line = line.substr(0, Math.min(MAX_DISPLAY_LENGTH, line.length));
        } else if (totalMatchLength > MAX_DISPLAY_LENGTH) {
            // impossible to display the whole match
            line = line.substr(ch, ch + MAX_DISPLAY_LENGTH);
            highlightOffset = ch;
        } else {
            // Try to have both beginning and end of match displayed
            padding = MAX_DISPLAY_LENGTH - totalMatchLength;
            rightPadding = Math.floor(Math.min(padding / 2, line.length - highlightEndCh));
            leftPadding = Math.ceil(padding - rightPadding);
            highlightOffset = ch - leftPadding;
            line = line.substring(highlightOffset, highlightEndCh + rightPadding);
        }

        matches.push({
            start:       {line: lineNum, ch: ch},
            end:         {line: lineNum + numMatchedLines - 1, ch: endCh},

            highlightOffset: highlightOffset,

            // Note that the following offsets from the beginning of the file are *not* updated if the search
            // results change. These are currently only used for multi-file replacement, and we always
            // abort the replace (by shutting the results panel) if we detect any result changes, so we don't
            // need to keep them up to date. Eventually, we should either get rid of the need for these (by
            // doing everything in terms of line/ch offsets, though that will require re-splitting files when
            // doing a replace) or properly update them.
            startOffset: match.index,
            endOffset:   match.index + totalMatchLength,

            line:        line,
            result:      match,
            isChecked:   true
        });

        // We have the max hits in just this 1 file. Stop searching this file.
        // This fixed issue #1829 where code hangs on too many hits.
        // Adds one over MAX_RESULTS_IN_A_FILE in order to know if the search has exceeded
        // or is equal to MAX_RESULTS_IN_A_FILE. Additional result removed in SearchModel
        if (matches.length > MAX_RESULTS_IN_A_FILE) {
            queryExpr.lastIndex = 0;
            break;
        }

        // Pathological regexps like /^/ return 0-length matches. Ensure we make progress anyway
        if (totalMatchLength === 0) {
            queryExpr.lastIndex++;
        }
    }

    return matches;
}

/**
 * Clears the cached file contents of the project
 */
function clearProjectCache() {
    projectCache = [];
}


/**
 * Gets the file size in bytes.
 * @param   {string} fileName The name of the file to get the size
 * @returns {Number} the file size in bytes
 */
function getFilesizeInBytes(fileName) {
    try {
        var stats = fs.statSync(fileName);
        return stats.size || 0;
    } catch (ex) {
        console.log(ex);
        return 0;
    }
}

/**
 * Get the contents of a file from cache given the path. Also adds the file contents to cache from disk if not cached.
 * Will not read/cache files greater than MAX_FILE_SIZE_TO_INDEX in size.
 * @param   {string} filePath full file path
 * @return {string} contents or null if no contents
 */
function getFileContentsForFile(filePath) {
    if (projectCache[filePath] || projectCache[filePath] === "") {
        return projectCache[filePath];
    }
    try {
        if (getFilesizeInBytes(filePath) <= MAX_FILE_SIZE_TO_INDEX) {
            projectCache[filePath] = fs.readFileSync(filePath, 'utf8');
        } else {
            projectCache[filePath] = "";
        }
    } catch (ex) {
        console.log(ex);
        projectCache[filePath] = null;
    }
    return projectCache[filePath];
}

/**
 * Sets the list of matches for the given path, removing the previous match info, if any, and updating
 * the total match count. Note that for the count to remain accurate, the previous match info must not have
 * been mutated since it was set.
 * @param {string} fullpath Full path to the file containing the matches.
 * @param {!{matches: Object, collapsed: boolean=}} resultInfo Info for the matches to set:
 *      matches - Array of matches, in the format returned by FindInFiles.getSearchMatches()
 *      collapsed - Optional: whether the results should be collapsed in the UI (default false).
 */
function setResults(fullpath, resultInfo, maxResultsToReturn) {
    if (results[fullpath]) {
        numMatches -= results[fullpath].matches.length;
        delete results[fullpath];
    }

    if (foundMaximum || !resultInfo || !resultInfo.matches || !resultInfo.matches.length) {
        return;
    }

    // Make sure that the optional `collapsed` property is explicitly set to either true or false,
    // to avoid logic issues later with comparing values.
    resultInfo.collapsed = collapseResults;

    results[fullpath] = resultInfo;
    numMatches += resultInfo.matches.length;
    evaluatedMatches += resultInfo.matches.length;
    maxResultsToReturn = maxResultsToReturn || MAX_RESULTS_TO_RETURN;
    if (numMatches >= maxResultsToReturn || evaluatedMatches > MAX_TOTAL_RESULTS) {
        foundMaximum = true;
    }
}

/**
 * Finds search results in the given file and adds them to 'results'
 * @param {string} filepath
 * @param {string} text   contents of the file
 * @param {Object} queryExpr
 * @param {number} maxResultsToReturn the maximum of results that should be returned in the current search.
 */
function doSearchInOneFile(filepath, text, queryExpr, maxResultsToReturn) {
    var matches = getSearchMatches(text, queryExpr);
    setResults(filepath, {matches: matches}, maxResultsToReturn);
}

/**
 * Search in the list of files given and populate the results
 * @param {array} fileList           array of file paths
 * @param {Object} queryExpr
 * @param {number} startFileIndex    the start index of the array from which the search has to be done
 * @param {number} maxResultsToReturn  the maximum number of results to return in this search
 */
function doSearchInFiles(fileList, queryExpr, startFileIndex, maxResultsToReturn) {
    var i;
    if (fileList.length === 0) {
        console.log('no files found');
        return;

    } else {
        startFileIndex = startFileIndex || 0;
        for (i = startFileIndex; i < fileList.length && !foundMaximum; i++) {
            doSearchInOneFile(fileList[i], getFileContentsForFile(fileList[i]), queryExpr, maxResultsToReturn);
        }
        lastSearchedIndex = i;
    }
}

// Copied from StringUtils.js
function regexEscape(str) {
    return str.replace(/([.?*+\^$\[\]\\(){}|\-])/g, "\\$1");
}

/**
 * Parses the given query into a regexp, and returns whether it was valid or not.
 * @param {{query: string, caseSensitive: boolean, isRegexp: boolean}} queryInfo
 * @return {{queryExpr: RegExp, valid: boolean, empty: boolean, error: string}}
 *      queryExpr - the regexp representing the query
 *      valid - set to true if query is a nonempty string or a valid regexp.
 *      empty - set to true if query was empty.
 *      error - set to an error string if valid is false and query is nonempty.
 */
function parseQueryInfo(queryInfo) {
    var queryExpr;

    // TODO: only major difference between this one and the one in FindReplace is that
    // this always returns a regexp even for simple strings. Reconcile.
    if (!queryInfo || !queryInfo.query) {
        return {empty: true};
    }

    // For now, treat all matches as multiline (i.e. ^/$ match on every line, not the whole
    // document). This is consistent with how single-file find works. Eventually we should add
    // an option for this.
    var flags = "gm";
    if (!queryInfo.isCaseSensitive) {
        flags += "i";
    }

    // Is it a (non-blank) regex?
    if (queryInfo.isRegexp) {
        try {
            queryExpr = new RegExp(queryInfo.query, flags);
        } catch (e) {
            return {valid: false, error: e.message};
        }
    } else {
        // Query is a plain string. Turn it into a regexp
        queryExpr = new RegExp(regexEscape(queryInfo.query), flags);
    }
    return {valid: true, queryExpr: queryExpr};
}

/**
 * Crawls through the files in the project ans stores them in cache. Since that could take a while
 * we do it in batches so that node wont be blocked.
 */
function fileCrawler() {
    if (!files || (files && files.length === 0)) {
        setTimeout(fileCrawler, 1000);
        return;
    }
    var contents = "";
    if (currentCrawlIndex < files.length) {
        contents = getFileContentsForFile(files[currentCrawlIndex]);
        if (contents) {
            cacheSize += contents.length;
        }
        currentCrawlIndex++;
    }
    if (currentCrawlIndex < files.length) {
        crawlComplete = false;
        setImmediate(fileCrawler);
    } else {
        crawlComplete = true;
        if (!crawlEventSent) {
            crawlEventSent = true;
            _domainManager.emitEvent("FindInFiles", "crawlComplete", [files.length, cacheSize]);
        }
        setTimeout(fileCrawler, 1000);
    }
}

/**
 * Init for project, resets the old project cache, and sets the crawler function to
 * restart the file crawl
 * @param   {array} fileList an array of files
 */
function initCache(fileList) {
    files = fileList;
    currentCrawlIndex = 0;
    cacheSize = 0;
    clearProjectCache();
    crawlEventSent = false;
}

/**
 * Counts the number of matches matching the queryExpr in the given contents
 * @param   {String} contents  The contents to search on
 * @param   {Object} queryExpr
 * @return {number} number of matches
 */
function countNumMatches(contents, queryExpr) {
    if (!contents) {
        return 0;
    }
    var matches = contents.match(queryExpr);
    return matches ? matches.length : 0;
}

/**
 * Get the total number of matches from all the files in fileList
 * @param   {array} fileList  file path array
 * @param   {Object} queryExpr
 * @return {Number} total number of matches
 */
function getNumMatches(fileList, queryExpr) {
    var i,
        matches = 0;
    for (i = 0; i < fileList.length; i++) {
        var temp = countNumMatches(getFileContentsForFile(fileList[i]), queryExpr);
        if (temp) {
            numFiles++;
            matches += temp;
        }
        if (matches > MAX_TOTAL_RESULTS) {
            exceedsMaximum = true;
            break;
        }
    }
    return matches;
}

/**
 * Do a search with the searchObject context and return the results
 * @param   {Object}   searchObject
 * @param   {boolean} nextPages    set to true if to indicate that next page of an existing page is being fetched
 * @return {Object}   search results
 */
function doSearch(searchObject, nextPages) {

    savedSearchObject = searchObject;
    if (!files) {
        console.log("no file object found");
        return {};
    }
    results = {};
    numMatches = 0;
    numFiles = 0;
    foundMaximum = false;
    if (!nextPages) {
        exceedsMaximum = false;
        evaluatedMatches = 0;
    }
    var queryObject = parseQueryInfo(searchObject.queryInfo);
    if (searchObject.files) {
        files = searchObject.files;
    }
    if (searchObject.getAllResults) {
        searchObject.maxResultsToReturn = MAX_TOTAL_RESULTS;
    }
    doSearchInFiles(files, queryObject.queryExpr, searchObject.startFileIndex, searchObject.maxResultsToReturn);
    if (crawlComplete && !nextPages) {
        numMatches = getNumMatches(files, queryObject.queryExpr);
    }
    var send_object = {
        "results":  results,
        "foundMaximum":  foundMaximum,
        "exceedsMaximum":  exceedsMaximum
    };

    if (!nextPages) {
        send_object.numMatches = numMatches;
        send_object.numFiles = numFiles;
    }

    if (searchObject.getAllResults) {
        send_object.allResultsAvailable = true;
    }
    return send_object;
}

/**
 * Remove the list of given files from the project cache
 * @param   {Object}   updateObject
 */
function removeFilesFromCache(updateObject) {
    var fileList = updateObject.fileList || [],
        filesInSearchScope = updateObject.filesInSearchScope || [],
        i = 0;
    for (i = 0; i < fileList.length; i++) {
        delete projectCache[fileList[i]];
    }
    function isNotInRemovedFilesList(path) {
        return (filesInSearchScope.indexOf(path) === -1) ? true : false;
    }
    files = files ? files.filter(isNotInRemovedFilesList) : files;
}

/**
 * Adds the list of given files to the project cache. However the files will not be
 * read at this time. We just delete the project cache entry which will trigger a fetch on search.
 * @param   {Object}   updateObject
 */
function addFilesToCache(updateObject) {
    var fileList = updateObject.fileList || [],
        filesInSearchScope = updateObject.filesInSearchScope || [],
        i = 0,
        changedFilesAlreadyInList = [],
        newFiles = [];
    for (i = 0; i < fileList.length; i++) {
        // We just add a null entry indicating the precense of the file in the project list.
        // The file will be later read when required.
        projectCache[fileList[i]] = null;
    }

    //Now update the search scope
    function isInChangedFileList(path) {
        return (filesInSearchScope.indexOf(path) !== -1) ? true : false;
    }
    changedFilesAlreadyInList = files ? files.filter(isInChangedFileList) : [];
    function isNotAlreadyInList(path) {
        return (changedFilesAlreadyInList.indexOf(path) === -1) ? true : false;
    }
    newFiles = changedFilesAlreadyInList.filter(isNotAlreadyInList);
    files.push.apply(files, newFiles);
}

/**
 * Notification function on document changed, we update the cache with the contents
 * @param {Object} updateObject
 */
function documentChanged(updateObject) {
    projectCache[updateObject.filePath] = updateObject.docContents;
}

/**
 * Gets the next page of results of the ongoing search
 * @return {Object} search results
 */
function getNextPage() {
    var send_object = {
        "results":  {},
        "numMatches": 0,
        "foundMaximum":  foundMaximum,
        "exceedsMaximum":  exceedsMaximum
    };
    if (!savedSearchObject) {
        return send_object;
    }
    savedSearchObject.startFileIndex = lastSearchedIndex;
    return doSearch(savedSearchObject, true);
}

/**
 * Gets all the results for the saved search query if present or empty search results
 * @return {Object} The results object
 */
function getAllResults() {
    var send_object = {
        "results":  {},
        "numMatches": 0,
        "foundMaximum":  foundMaximum,
        "exceedsMaximum":  exceedsMaximum
    };
    if (!savedSearchObject) {
        return send_object;
    }
    savedSearchObject.startFileIndex = 0;
    savedSearchObject.getAllResults = true;
    return doSearch(savedSearchObject);
}

/**
 * Sets if the results should be collapsed
 * @param {boolean} collapse true to collapse
 */
function setCollapseResults(collapse) {
    collapseResults = collapse;
}

/**
 * Initialize the test domain with commands and events related to find in files.
 * @param {DomainManager} domainManager The DomainManager for the find in files domain "FindInFiles"
 */
function init(domainManager) {
    if (!domainManager.hasDomain("FindInFiles")) {
        domainManager.registerDomain("FindInFiles", {major: 0, minor: 1});
    }
    _domainManager = domainManager;
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "doSearch",    // command name
        doSearch,   // command handler function
        false,          // this command is synchronous in Node
        "Searches in project files and returns matches",
        [{name: "searchObject", // parameters
            type: "object",
            description: "Object containing search data"}],
        [{name: "searchResults", // return values
            type: "object",
            description: "Object containing results of the search"}]
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "nextPage",    // command name
        getNextPage,   // command handler function
        false,          // this command is synchronous in Node
        "get the next page of reults",
        [],
        [{name: "searchResults", // return values
            type: "object",
            description: "Object containing results of the search"}]
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "getAllResults",    // command name
        getAllResults,   // command handler function
        false,          // this command is synchronous in Node
        "get the next page of reults",
        [],
        [{name: "searchResults", // return values
            type: "object",
            description: "Object containing all results of the search"}]
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "collapseResults",    // command name
        setCollapseResults,   // command handler function
        false,          // this command is synchronous in Node
        "get the next page of reults",
        [{name: "collapse", // return values
            type: "boolean",
            description: "true to collapse"}],
        []
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "filesChanged",    // command name
        addFilesToCache,   // command handler function
        false,          // this command is synchronous in Node
        "files in the project has been changed, update cache",
        [{name: "updateObject", // parameters
            type: "object",
            description: "Object containing list of changed files"}],
        []
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "documentChanged",    // command name
        documentChanged,   // command handler function
        false,          // this command is synchronous in Node
        "informs that the document changed and updates the cache",
        [{name: "updateObject", // parameters
            type: "object",
            description: "update with the contents of the object"}],
        []
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "filesRemoved",    // command name
        removeFilesFromCache,   // command handler function
        false,          // this command is synchronous in Node
        "Searches in project files and returns matches",
        [{name: "updateObject", // parameters
            type: "object",
            description: "Object containing list of removed files"}],
        []
    );
    domainManager.registerCommand(
        "FindInFiles",       // domain name
        "initCache",    // command name
        initCache,   // command handler function
        false,          // this command is synchronous in Node
        "Caches the project for find in files in node",
        [{name: "fileList", // parameters
            type: "Array",
            description: "List of all project files - Path only"}],
        []
    );
    domainManager.registerEvent(
        "FindInFiles",     // domain name
        "crawlComplete",   // event name
        [
            {
                name: "numFiles",
                type: "number",
                description: "number of files cached"
            },
            {
                name: "cacheSize",
                type: "number",
                description: "The size of the file cache epressesd as string length of files"
            }
        ]
    );
    setTimeout(fileCrawler, 5000);
}

exports.init = init;