/
dictator
/
ragflow_sync
Обзор
Документация
Войти
/
dictator
/
ragflow_sync
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
CI/CD
Аналитика
master
cleanup/clean.py
191 строка
7 KB
Кряжев Сергей Владимирович
added cleanup stage
13 апр 2026, 12:19
13 апр 2026, 12:19
26ce166
Код
Авторство
О чём код?
import sys import re def remove_special_characters(text, special_chars=None): if special_chars is None: # Updated regex to include Cyrillic characters and preserve meaningful punctuation special_chars = r"[^\w\s\.,;:'\"?\!\-\u0400-\u04FF]" text = re.sub(special_chars, "", text) return text.strip() def remove_repeated_substrings(text, pattern=r"\.{2,}"): text = re.sub(pattern, ".", text) return text.strip() def remove_extra_spaces(text): # Preserve paragraph breaks and line structure while normalizing internal whitespace # First normalize existing double newlines to ensure consistency text = re.sub(r"\n\s*\n", "\n\n", text) # Convert single newlines to spaces but preserve paragraph breaks # Only convert non-newline whitespace to single spaces text = re.sub(r"[^\S\n]+", " ", text) # Remove leading whitespace from lines and normalize extra newlines lines = text.split("\n") cleaned_lines = [] for line in lines: # Don't skip empty lines, preserve meaningful paragraph breaks cleaned_lines.append(line.lstrip()) # Join with single newlines, but preserve paragraph breaks (double newlines) result = "\n".join(cleaned_lines) # Normalize multiple consecutive newlines to single newlines result = re.sub(r"\n{3,}", "\n\n", result) return result.strip() def remove_markdown_images(text): """Remove markdown image links like """ # Pattern to match markdown image links image_pattern = r"!\[[^\]]*\]\([^\)]*\)" text = re.sub(image_pattern, "", text) return text.strip() def find_and_process_long_urls(text): """ Identify and process URLs with 10+ percent-encoded sequences. Uses the grep pattern '(?:%[A-Fa-f0-9]{2}){10,}' to detect long encoded URLs and applies URL shortening logic from update_long_link.py. """ from urllib.parse import urlparse, parse_qs, urlencode, unquote def extract_jira_urls(text): """Extract Jira URLs more aggressively, handling malformed URLs with spaces.""" urls = [] # Find all sberworks.ru/jira URLs - more aggressive matching # Look for the pattern and extract up to &customfield_25100 or other known params pattern = r"https://sberworks\.ru/jira/[^&#]+" matches = re.findall(pattern, text) for match in matches: # Try to extend the URL to include known parameters # Look for the pattern and see if we can find more extended = match # Find where this match appears in the text pos = text.find(match) if pos >= 0: # Look for known parameter patterns after this rest = text[pos + len(match) :] # Find the customfield_25100 which is typically at the end custom_match = re.search(r"&customfield_25100=[a-f0-9-]+", rest) if custom_match: extended = match + rest[: custom_match.end()] urls.append(extended) return urls def extract_wiki_urls(text): """Extract Wiki URLs more aggressively.""" urls = [] # Look for the full Wiki URL pattern pattern = r"https://sberworks\.ru/wiki/pages/viewpage\.action\?[^&\s]+" matches = re.findall(pattern, text) urls.extend(matches) return urls def shorten_jira_url(url): """Shorten Jira URL by keeping only key parameters.""" try: parsed = urlparse(url) params = parse_qs(parsed.query) if "pid" not in params: return url new_params = {"pid": params.get("pid", [""])[0]} for key, value in params.items(): if key not in ["description", "summary"]: new_params[key] = value[0] new_query = urlencode(new_params, doseq=True) return parsed._replace(query=new_query).geturl() except Exception as e: # Log the exception instead of silently ignoring return url def shorten_wiki_url(url): """Shorten Wiki URL by keeping pageId and fragment.""" try: parsed = urlparse(url) params = parse_qs(parsed.query) if "pageId" not in params: return url new_params = {"pageId": params.get("pageId", [""])[0]} new_query = urlencode(new_params, doseq=True) fragment = parsed.fragment if fragment: fragment = unquote(fragment) result = parsed._replace(query=new_query, fragment=fragment).geturl() return result except Exception as e: # Log the exception instead of silently ignoring return url def process_url(url): """Process a URL to shorten if it's a long encoded URL.""" parsed = urlparse(url) path = parsed.path if "/jira/secure/CreateIssueDetails!init.jspa" in path: return shorten_jira_url(url) elif "/wiki/pages/viewpage.action" in path: return shorten_wiki_url(url) else: return url # Process text line by line - if a line has 10+ consecutive encoded sequences, # find and process the URL within it lines = text.split("\n") result_lines = [] for line in lines: # Check if line contains 10+ consecutive percent-encoded sequences if re.search(r"(?:%[A-Fa-f0-9]{2}){10,}", line): # Extract and process Jira and Wiki URLs for url in extract_jira_urls(line): shortened = process_url(url) line = line.replace(url, shortened) for url in extract_wiki_urls(line): shortened = process_url(url) line = line.replace(url, shortened) result_lines.append(line) return "\n".join(result_lines) def preprocess_text(text): # Remove repeated substrings like dots text = remove_repeated_substrings(text) # Find and process long encoded URLs BEFORE other cleaning steps # This must come first because remove_extra_spaces converts multiple spaces to single text = find_and_process_long_urls(text) # Remove extra spaces between lines and within lines text = remove_extra_spaces(text) # Remove markdown image links text = remove_markdown_images(text) # Additional cleaning steps can be added here return text.strip() if __name__ == "__main__": # Read input from stdin raw_text = sys.stdin.read() # Clean the text cleaned_text = preprocess_text(raw_text) # Print cleaned output to stdout print(cleaned_text)