Import Website Pages

Imports known URLs into a website source and returns the run. Poll Get Source Run with its id until state is completed, failed, or canceled.

Each URL is crawled on its own, and importing starts once every URL has been crawled. A URL that returns no page fails the run before any page is imported, and the source’s documents don’t change.

When the run completes, sourceUrls becomes the source’s page list. Each import sets the whole list, so send every URL you want the source to keep.

The source must be a web source. A source runs one sync, preview, or import at a time. Wait for its active run to end before starting another, and cancel a crawl preview you won’t import.

For both ways to import website pages, see Import Website Pages.

1 min read

post/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import

Request example

curl --request POST \
  --url https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import \
  --header 'Authorization: Bearer <token>' \
  --header 'Idempotency-Key: <string>' \
  --header 'Content-Type: application/json' \
  --data '{
  "replaceExistingSelection": true,
  "sourceUrls": [
    "https://docs.example.com/guides/billing",
    "https://docs.example.com/pricing"
  ]
}'

import requests

url = "https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
headers = {
    "Authorization": "Bearer <token>",
    "Idempotency-Key": "<string>",
    "Content-Type": "application/json",
}
payload = {
    "replaceExistingSelection": True,
    "sourceUrls": [
        "https://docs.example.com/guides/billing",
        "https://docs.example.com/pricing"
    ]
}

response = requests.request("POST", url, headers=headers, json=payload)

print(response.text)

const response = await fetch('https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import', {
  method: 'POST',
  headers: {
    'Authorization': 'Bearer <token>',
    'Idempotency-Key': '<string>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    "replaceExistingSelection": true,
    "sourceUrls": [
      "https://docs.example.com/guides/billing",
      "https://docs.example.com/pricing"
    ]
  })
});

const data = await response.json();
console.log(data);

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => 'https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import',
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => 'POST',
  CURLOPT_POSTFIELDS => '{
  "replaceExistingSelection": true,
  "sourceUrls": [
    "https://docs.example.com/guides/billing",
    "https://docs.example.com/pricing"
  ]
}',
  CURLOPT_HTTPHEADER => [
    'Authorization: Bearer <token>',
    'Idempotency-Key: <string>',
    'Content-Type: application/json',
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

package main

import (
	"fmt"
	"io"
	"net/http"
	"strings"
)

func main() {
	url := "https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"
	payload := strings.NewReader("{\n  \"replaceExistingSelection\": true,\n  \"sourceUrls\": [\n    \"https://docs.example.com/guides/billing\",\n    \"https://docs.example.com/pricing\"\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Idempotency-Key", "<string>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))
}

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import"))
    .header("Authorization", "Bearer <token>")
    .header("Idempotency-Key", "<string>")
    .header("Content-Type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"replaceExistingSelection\": true,\n  \"sourceUrls\": [\n    \"https://docs.example.com/guides/billing\",\n    \"https://docs.example.com/pricing\"\n  ]\n}"))
    .build();

HttpResponse<String> response = HttpClient.newHttpClient()
    .send(request, HttpResponse.BodyHandlers.ofString());

System.out.println(response.body());

require 'uri'
require 'net/http'

url = URI('https://api.dify.ai/v1/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import')

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = url.scheme == 'https'

request = Net::HTTP::Post.new(url)
request['Authorization'] = 'Bearer <token>'
request['Idempotency-Key'] = '<string>'
request['Content-Type'] = 'application/json'
request.body = '{
  "replaceExistingSelection": true,
  "sourceUrls": [
    "https://docs.example.com/guides/billing",
    "https://docs.example.com/pricing"
  ]
}'

response = http.request(request)
puts response.read_body

Response

{
  "canceled_at": null,
  "checkpoint": "queued",
  "completed_at": null,
  "created_at": "2026-10-20T10:00:00Z",
  "cursor": null,
  "execution_attempts": 0,
  "failure": null,
  "id": "e1f2a3b4-c5d6-4e7f-8a9b-0c1d2e3f4a5b",
  "kind": "crawl-import",
  "knowledge_space_id": "0a9d3c2e-5b6f-4e1a-8c7d-2f3e4a5b6c7d",
  "last_error_code": null,
  "max_execution_attempts": 5,
  "progress_completed": 0,
  "progress_failed": 0,
  "progress_skipped": 0,
  "progress_total": 2,
  "source_id": "6f1c2b0e-3d0b-4a5e-9f4c-1c0e6a7b8d90",
  "state": "queued",
  "updated_at": "2026-10-20T10:00:00Z"
}

{
  "code": "knowledge_fs_invalid_request",
  "message": "Agent Knowledge Base request is invalid.",
  "status": 400,
  "violations": [
    {
      "field": [
        "sourceUrls"
      ],
      "type": "too_long"
    }
  ]
}

{
  "code": "knowledge_fs_invalid_request",
  "message": "Agent Knowledge Base request is invalid.",
  "status": 400
}

{
  "code": "knowledge_fs_invalid_request",
  "failure": {
    "category": "validation",
    "code": "KNOWLEDGE_FS_INVALID_REQUEST",
    "message": "The Agent Knowledge Base request is invalid.",
    "retryPolicy": "never",
    "traceId": "7f3c1a90-2e64-4b7d-8a51-6c0d9e2f4b83"
  },
  "message": "The Agent Knowledge Base request is invalid.",
  "status": 400
}

{
  "code": "knowledge_fs_invalid_credential",
  "message": "Invalid Agent Knowledge Base service credential.",
  "status": 401
}

{
  "code": "unauthorized",
  "message": "Access token is invalid",
  "status": 401
}

{
  "code": "forbidden",
  "message": "Sorry, you have reached the knowledge base request rate limit of your subscription.",
  "status": 403
}

{
  "code": "knowledge_fs_access_denied",
  "message": "Agent Knowledge Base operation is not allowed.",
  "status": 403
}

{
  "code": "knowledge_fs_resource_not_found",
  "failure": {
    "category": "not_found",
    "code": "KNOWLEDGE_FS_NOT_FOUND",
    "message": "The requested Agent Knowledge Base resource was not found.",
    "retryPolicy": "never",
    "traceId": "7f3c1a90-2e64-4b7d-8a51-6c0d9e2f4b83"
  },
  "message": "The requested Agent Knowledge Base resource was not found.",
  "status": 404
}

{
  "code": "not_found",
  "message": "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.",
  "status": 404
}

{
  "code": "knowledge_fs_conflict",
  "failure": {
    "action": "retry",
    "category": "conflict",
    "code": "KNOWLEDGE_FS_CONFLICT",
    "message": "The Agent Knowledge Base operation conflicts with the current resource state.",
    "retryPolicy": "manual",
    "traceId": "7f3c1a90-2e64-4b7d-8a51-6c0d9e2f4b83"
  },
  "message": "The Agent Knowledge Base operation conflicts with the current resource state.",
  "status": 409
}

{
  "code": "knowledge_fs_request_too_large",
  "message": "Agent Knowledge Base request is too large.",
  "status": 413
}

{
  "code": "knowledge_fs_upstream_unavailable",
  "failure": {
    "action": "contact_admin",
    "category": "internal",
    "code": "KNOWLEDGE_FS_INTERNAL_ERROR",
    "message": "Agent Knowledge Base could not complete the operation. Try again, or contact an administrator with the error reference.",
    "retryPolicy": "manual",
    "traceId": "5e2a9c17-3b4d-4f60-8a1b-2c3d4e5f6a7b"
  },
  "message": "Agent Knowledge Base could not complete the operation. Try again, or contact an administrator with the error reference.",
  "status": 502
}

{
  "code": "knowledge_fs_upstream_unavailable",
  "message": "Agent Knowledge Base is unavailable.",
  "status": 502
}

{
  "code": "knowledge_fs_operation_unavailable",
  "message": "Agent Knowledge Base operation is not available.",
  "status": 503
}

{
  "code": "knowledge_fs_operation_unavailable",
  "failure": {
    "action": "retry",
    "category": "dependency",
    "code": "KNOWLEDGE_FS_UNAVAILABLE",
    "message": "A service required by Agent Knowledge Base is temporarily unavailable.",
    "retryPolicy": "manual",
    "traceId": "5e2a9c17-3b4d-4f60-8a1b-2c3d4e5f6a7b"
  },
  "message": "A service required by Agent Knowledge Base is temporarily unavailable.",
  "status": 503
}

post/knowledge-fs/spaces/{knowledge_base_id}/sources/{source_id}/crawl-import

Authorization

Authorizationstringheaderrequired#

Every request authenticates with an API key: Authorization: Bearer {API_KEY}. App endpoints take an app API key; knowledge endpoints take a knowledge base API key (Get Started).

Keep keys server-side; never embed them in client code. Requests with a missing or invalid key fail with HTTP 401 (unauthorized).

Path parameters

knowledge_base_idstringrequired#

Knowledge base ID: the last segment of the Service API endpoint shown in the knowledge base's API Access dialog (…/knowledge-fs/spaces/<knowledge base ID>), and the segment after /datasets/new/ in the address bar.

source_idstringrequired#

Source ID. From List Sources.

Headers

Idempotency-Keystringrequired#

Repeating the same key and request from the same API key returns the original run in its current state. Use a new key for each import you start. Reusing the key on this source with a different body returns 409. Use printable ASCII characters without leading or trailing whitespace.

Length: 8–255

Body

application/json

replaceExistingSelectionboolean#

Set to true to delete, in the same run, the source's documents for pages not in sourceUrls.

Default: false

sourceUrlsstring[]required#

URLs to import, each listed once. Leading and trailing whitespace is trimmed.

Items: 1–200

Responses

application/json

The run, with kind: crawl-import, state: queued, and progress_total set to the number of URLs.

canceled_atstring#

When the run was canceled. null unless state is canceled.

checkpointstring#

Last step the run reached: queued, provider-read, preview-staged, selection-frozen, materialized, cleanup-staging, or source-committed. Retry Source Run starts the run over whatever its checkpoint.

completed_atstring#

When the run ended (completed, zero_results, failed, or canceled), ISO 8601. null while it runs.

created_atstring#

Creation time, ISO 8601.

cursorstring#

Provider cursor the run resumes from. null when the run has not paged.

execution_attemptsinteger#

Times a worker has picked up the run, including after Retry Source Run, which does not reset it.

failureobject#

Error details when state is failed. stage is the run's checkpoint at the failure, and traceId is the run ID. null otherwise.

Show child attributes

actionstring#

Suggested next step, when there is one: configure_model, configure_parser, configure_source, contact_admin, reupload, or retry. null when not applicable; omitted from HTTP error bodies.

categorystring#

Kind of failure: authorization, canceled, configuration, conflict, dependency, internal, not_found, rate_limit, timeout, or validation.

codestring#

Stable error code in upper snake case, for example KNOWLEDGE_FS_CONFLICT or SOURCE_CREDENTIAL_TEST_FAILED.

messagestring#

Public message for the error code or failure category. Provider error details are not exposed.

parametersobject#

Up to 8 values that qualify the failure, with keys among attempt, currentRowVersion, documentCount, expectedRowVersion, fileSizeBytes, limit, maxFileSizeBytes, maxItems, modelType, providerKind, retryAfterSeconds, and status. null when there are none; omitted from HTTP error bodies.

retryPolicystring#

Whether a retry can help: automatic (the service retries on its own), manual (retry yourself), after_configuration (fix the knowledge base's setup in Dify first), or never (change the request).

stagestring#

Stage the failure belongs to, for example query_generation or credential-test. null when not applicable; omitted from HTTP error bodies.

traceIdstring#

Reference to quote when contacting an administrator. May be null in a successful response containing failure details, or omitted from an HTTP error body.

idstring#

Run ID. Pass it to Get Source Run, Cancel Source Run, or Retry Source Run.

kindstring#

What the run does:

sync: started by Sync Source, Update Source, a schedule, or Dify.
crawl-preview: started by Preview Website Crawl. It keeps this kind while Import Selected Pages imports from it.
crawl-import: started by Import Website Pages.
online-document-import, online-drive-import: started by Import Source Items.

Dify also starts previews and imports when a source is added or edited there.

knowledge_space_idstring#

Internal ID of the knowledge base. Not the knowledge base ID in the path.

last_error_codestring#

failure.code when state is failed. null otherwise.

max_execution_attemptsinteger#

Limit on execution_attempts. Once it is reached, Retry Source Run answers 409.

progress_completedinteger#

Items finished, including unchanged items a sync checked.

progress_failedinteger#

Always 0 for source runs: a failing item fails the whole run.

progress_skippedinteger#

Always 0 for source runs.

progress_totalinteger#

Items the run expects to process: the pages or items an import requested, the pages a crawl preview has staged so far, or the source's items once a sync has listed them. null until known.

While an Import Website Pages run crawls its URLs, progress_completed and progress_total both count the pages crawled so far. Once every URL is crawled, they restart at 0 of the number of URLs.

source_idstring#

Source the run belongs to.

statestring#

Progress of the run.

Live states:

queued: waiting for a worker
crawling: a crawl preview is fetching pages
preview_ready: a crawl preview is waiting for Import Selected Pages or Cancel Source Run
importing: importing and indexing items
syncing: refreshing the source's documents

Terminal states:

completed: every item was imported or refreshed, indexed, and made active
zero_results: a crawl preview found no pages
failed: see failure
canceled

After failed or canceled, Retry Source Run runs it again, and Sync Source starts a new sync.

updated_atstring#

Last change, ISO 8601.

Import Selected Pages Import Source Items