app: description: '' icon: 🤖 icon_background: '#FFEAD5' icon_type: emoji mode: workflow name: Scrape.do Web Scraper use_icon_as_answer_icon: false dependencies: [] kind: app version: 0.6.0 workflow: conversation_variables: [] environment_variables: [] features: file_upload: allowed_file_extensions: - .JPG - .JPEG - .PNG - .GIF - .WEBP - .SVG allowed_file_types: - image allowed_file_upload_methods: - local_file - remote_url enabled: false fileUploadConfig: attachment_image_file_size_limit: 2 audio_file_size_limit: 50 batch_count_limit: 5 file_size_limit: 15 file_upload_limit: 50 image_file_batch_limit: 10 image_file_size_limit: 10 single_chunk_attachment_limit: 10 video_file_size_limit: 100 workflow_file_upload_limit: 10 image: enabled: false number_limits: 3 transfer_methods: - local_file - remote_url number_limits: 3 opening_statement: '' retriever_resource: enabled: true sensitive_word_avoidance: enabled: false speech_to_text: enabled: false suggested_questions: [] suggested_questions_after_answer: enabled: false text_to_speech: enabled: false language: '' voice: '' graph: edges: - data: isInIteration: false isInLoop: false sourceType: http-request targetType: document-extractor id: 1783504782065-source-1783505277324-target selected: false source: '1783504782065' sourceHandle: source target: '1783505277324' targetHandle: target type: custom zIndex: 0 - data: isInIteration: false isInLoop: false sourceType: document-extractor targetType: code id: 1783505277324-source-1783506040837-target selected: false source: '1783505277324' sourceHandle: source target: '1783506040837' targetHandle: target type: custom zIndex: 0 - data: isInIteration: false isInLoop: false sourceType: code targetType: iteration id: 1783506040837-source-1783506531638-target source: '1783506040837' sourceHandle: source target: '1783506531638' targetHandle: target type: custom zIndex: 0 - data: isInIteration: true isInLoop: false iteration_id: '1783506531638' sourceType: iteration-start targetType: http-request id: 1783506531638start-source-1783506623202-target source: 1783506531638start sourceHandle: source target: '1783506623202' targetHandle: target type: custom zIndex: 1002 - data: isInIteration: false isInLoop: false sourceType: iteration targetType: end id: 1783506531638-source-1783508642624-target source: '1783506531638' sourceHandle: source target: '1783508642624' targetHandle: target type: custom zIndex: 0 - data: isInLoop: false sourceType: start targetType: http-request id: 1783510875696-source-1783504782065-target source: '1783510875696' sourceHandle: source target: '1783504782065' targetHandle: target type: custom zIndex: 0 nodes: - data: selected: false title: User Input type: start variables: [] height: 73 id: '1783510875696' position: x: 200.85780458886472 y: 223 positionAbsolute: x: 200.85780458886472 y: 223 selected: false sourcePosition: right targetPosition: left type: custom width: 242 - data: authorization: config: null type: no-auth body: data: [] type: none headers: '' method: get params: '' retry_config: max_retries: 3 retry_enabled: true retry_interval: 100 selected: false ssl_verify: true timeout: max_connect_timeout: 0 max_read_timeout: 0 max_write_timeout: 0 title: HTTP Request type: http-request url: https://docs.google.com/spreadsheets/d/1_QOL88zR4v9izZP22ZCFqD8V4VrI77al8-Ds79R6Uqg/gviz/tq?tqx=out:csv variables: [] height: 169 id: '1783504782065' position: x: 659.9812754850634 y: 244 positionAbsolute: x: 659.9812754850634 y: 244 selected: true sourcePosition: right targetPosition: left type: custom width: 242 - data: is_array_file: true selected: false title: Doc Extractor type: document-extractor variable_selector: - '1783504782065' - files height: 104 id: '1783505277324' position: x: 965 y: 244 positionAbsolute: x: 965 y: 244 selected: false sourcePosition: right targetPosition: left type: custom width: 242 - data: code: "def main(text) -> dict:\n # Doc Extractor may return a list of strings\ \ or a single string\n if isinstance(text, list):\n text = \"\\\ n\".join(text)\n urls = []\n for line in text.splitlines():\n \ \ line = line.strip().strip('\"').strip(\",\")\n if line.startswith(\"\ http\"):\n urls.append(line)\n return {\"url_list\": urls}" code_language: python3 outputs: url_list: children: null type: array[string] selected: false title: Code type: code variables: - value_selector: - '1783505277324' - text value_type: array[string] variable: text height: 52 id: '1783506040837' position: x: 1267.6769734152942 y: 244 positionAbsolute: x: 1267.6769734152942 y: 244 selected: false sourcePosition: right targetPosition: left type: custom width: 242 - data: error_handle_mode: terminated flatten_output: true height: 209 is_parallel: false iterator_input_type: array[string] iterator_selector: - '1783506040837' - url_list output_selector: - '1783506623202' - body output_type: array[string] parallel_nums: 10 selected: false start_node_id: 1783506531638start title: Iteration type: iteration width: 386 height: 209 id: '1783506531638' position: x: 1573.3208922477843 y: 244 positionAbsolute: x: 1573.3208922477843 y: 244 selected: false sourcePosition: right targetPosition: left type: custom width: 386 zIndex: 1 - data: desc: '' isInIteration: true selected: false title: '' type: iteration-start draggable: false height: 48 id: 1783506531638start parentId: '1783506531638' position: x: 24 y: 68 positionAbsolute: x: 1597.3208922477843 y: 312 selectable: false selected: false sourcePosition: right targetPosition: left type: custom-iteration-start width: 44 zIndex: 1002 - data: authorization: config: null type: no-auth body: data: [] type: none headers: '' isInIteration: true isInLoop: false iteration_id: '1783506531638' method: get params: 'token:1d29ea17366d4b68867fa0b2206265d0c242553231f url:{{#1783506531638.item#}}' retry_config: max_retries: 3 retry_enabled: true retry_interval: 100 selected: false ssl_verify: true timeout: max_connect_timeout: 0 max_read_timeout: 0 max_write_timeout: 0 title: HTTP Request 2 type: http-request url: https://api.scrape.do/ variables: [] height: 121 id: '1783506623202' parentId: '1783506531638' position: x: 128 y: 68 positionAbsolute: x: 1701.3208922477843 y: 312 selected: false sourcePosition: right targetPosition: left type: custom width: 242 zIndex: 1002 - data: outputs: - value_selector: - '1783506531638' - output value_type: array[string] variable: result selected: false title: Output type: end height: 88 id: '1783508642624' position: x: 2077.0081395920797 y: 244 positionAbsolute: x: 2077.0081395920797 y: 244 selected: false sourcePosition: right targetPosition: left type: custom width: 242 viewport: x: -65.39046899971765 y: 107.3442939653591 zoom: 0.504725368854005 rag_pipeline_variables: []