Appearance
SOP: On-Page SEO Audit Fresh
The On-Page API crawls your site like a search engine would. You post a task, the crawl runs asynchronously, and then you pull the results. This covers the full process.
Step 1: Set Up a Crawl Task
Post a crawl task for the target URL. The API will spider the site and flag issues.
python
import os
import requests
from requests.auth import HTTPBasicAuth
import time
import json
auth = HTTPBasicAuth(
os.environ["DATAFORSEO_LOGIN"],
os.environ["DATAFORSEO_PASSWORD"]
)
target_url = "https://example.com"
payload = [
{
"target": target_url,
"max_crawl_pages": 100, # limit to 100 pages for cost control
"start_url": target_url,
"crawl_delay": 2, # seconds between requests (be polite)
"store_raw_html": False, # set True if you need the HTML
"enable_content_parsing": True, # parse headings, word count
"check_spell": False, # skip spell check (costs more)
"calculate_keyword_density": False,
"custom_js": None,
"respect_sitemap": True,
"enable_javascript": False, # set True for JS-heavy sites (costs more)
"browser_preset": None
}
]
response = requests.post(
"https://api.dataforseo.com/v3/on_page/task_post",
auth=auth,
json=payload
)
data = response.json()
task = data["tasks"][0]
task_id = task["id"]
print(f"Crawl task created: {task_id}")
print(f"Status: {task['status_message']}")
# Save task ID
with open("crawl_task_id.txt", "w") as f:
f.write(task_id)Step 2: Wait for Crawl to Complete
Poll the summary endpoint until the crawl is done.
python
def wait_for_crawl(auth, task_id, max_minutes=15):
"""Poll until crawl is complete or timeout."""
deadline = time.time() + (max_minutes * 60)
while time.time() < deadline:
response = requests.get(
f"https://api.dataforseo.com/v3/on_page/summary/{task_id}",
auth=auth
)
data = response.json()
task = data["tasks"][0]
if task["status_code"] != 20000:
print(f"Task error: {task['status_message']}")
return None
result = task["result"][0]
crawl_status = result.get("crawl_progress", "")
pages_crawled = result.get("pages_crawled", 0)
print(f"Status: {crawl_status} | Pages crawled: {pages_crawled}")
if crawl_status == "finished":
return result
time.sleep(15)
print("Timeout waiting for crawl")
return None
summary = wait_for_crawl(auth, task_id)
if summary:
print(f"\nCrawl complete!")
print(f"Pages crawled: {summary['pages_crawled']}")
print(f"Pages with errors: {summary.get('pages_crawled_with_errors', 0)}")Step 3: Pull Page-Level Issues
Fetch the list of crawled pages with their on-page data.
python
def get_crawled_pages(auth, task_id, limit=100):
"""Get page data for crawled URLs."""
payload = [
{
"id": task_id,
"limit": limit,
"filters": [
["status_code", "=", 200] # only live pages
]
}
]
response = requests.post(
"https://api.dataforseo.com/v3/on_page/pages",
auth=auth,
json=payload
)
data = response.json()
task = data["tasks"][0]
return task["result"][0].get("items", [])
pages = get_crawled_pages(auth, task_id, limit=100)
print(f"\nPages retrieved: {len(pages)}")Step 4: Analyze Issues
Parse each page for the most common on-page SEO problems.
python
def analyze_page_issues(pages):
"""Categorize on-page issues across all crawled pages."""
issues = {
"missing_title": [],
"duplicate_title": [],
"title_too_long": [], # > 60 chars
"title_too_short": [], # < 30 chars
"missing_meta_desc": [],
"meta_desc_too_long": [], # > 160 chars
"missing_h1": [],
"multiple_h1": [],
"low_word_count": [], # < 300 words
"broken_links": [],
"slow_load": [], # > 3 seconds
"no_canonical": [],
"non_indexable": []
}
for page in pages:
url = page.get("url", "")
meta = page.get("meta", {})
title = meta.get("title", "")
description = meta.get("description", "")
h1_count = len(meta.get("htags", {}).get("h1", []))
word_count = meta.get("content", {}).get("plain_text_word_count", 0)
load_time = page.get("page_timing", {}).get("duration_time", 0)
# Title checks
if not title:
issues["missing_title"].append(url)
elif len(title) > 60:
issues["title_too_long"].append({"url": url, "title": title, "length": len(title)})
elif len(title) < 30:
issues["title_too_short"].append({"url": url, "title": title, "length": len(title)})
# Meta description checks
if not description:
issues["missing_meta_desc"].append(url)
elif len(description) > 160:
issues["meta_desc_too_long"].append({"url": url, "length": len(description)})
# H1 checks
if h1_count == 0:
issues["missing_h1"].append(url)
elif h1_count > 1:
issues["multiple_h1"].append({"url": url, "count": h1_count})
# Content check
if word_count < 300:
issues["low_word_count"].append({"url": url, "word_count": word_count})
# Load time
if load_time > 3000: # milliseconds
issues["slow_load"].append({"url": url, "load_ms": load_time})
# Indexability
if not meta.get("follow") or not meta.get("index"):
issues["non_indexable"].append(url)
return issues
issues = analyze_page_issues(pages)Step 5: Check for Broken Links
Separately pull the broken links report.
python
def get_broken_links(auth, task_id):
"""Get all broken links found during the crawl."""
payload = [
{
"id": task_id,
"filters": [
["status_code", ">", 399] # 4xx and 5xx
],
"limit": 100
}
]
response = requests.post(
"https://api.dataforseo.com/v3/on_page/links",
auth=auth,
json=payload
)
data = response.json()
task = data["tasks"][0]
items = task["result"][0].get("items", [])
broken = []
for link in items:
broken.append({
"url_from": link.get("page_from"),
"url_to": link.get("link"),
"status_code": link.get("status_code"),
"anchor": link.get("anchor")
})
return broken
broken_links = get_broken_links(auth, task_id)
print(f"Broken links found: {len(broken_links)}")Step 6: Priority Fix Report
Rank issues by SEO impact and output a prioritized fix list.
python
def print_priority_report(issues, broken_links):
"""Print prioritized fix list."""
print("\n" + "=" * 60)
print("ON-PAGE SEO AUDIT — PRIORITY FIXES")
print("=" * 60)
priorities = [
("CRITICAL", [
("Missing title tags", len(issues["missing_title"])),
("Missing H1 tags", len(issues["missing_h1"])),
("Broken links (4xx/5xx)", len(broken_links)),
("Non-indexable pages", len(issues["non_indexable"])),
]),
("HIGH", [
("Missing meta descriptions", len(issues["missing_meta_desc"])),
("Pages under 300 words", len(issues["low_word_count"])),
("Multiple H1 tags", len(issues["multiple_h1"])),
]),
("MEDIUM", [
("Title too long (>60 chars)", len(issues["title_too_long"])),
("Title too short (<30 chars)", len(issues["title_too_short"])),
("Meta desc too long (>160)", len(issues["meta_desc_too_long"])),
("Slow pages (>3s)", len(issues["slow_load"])),
])
]
for level, items in priorities:
print(f"\n[{level}]")
for label, count in items:
if count > 0:
print(f" {label}: {count} pages")
print_priority_report(issues, broken_links)Common Issues Reference
| Issue | SEO Impact | Fix |
|---|---|---|
| Missing title tag | Critical | Add unique, descriptive titles under 60 chars |
| Missing H1 | High | Add one H1 per page matching the target keyword |
| Duplicate title | High | Rewrite each title to be unique |
| Missing meta description | Medium | Write 120-155 char descriptions with CTA |
| Low word count | Medium | Expand thin content or consolidate pages |
| Broken internal links | High | Fix or redirect the broken URLs |
| Slow page (>3s) | High | Compress images, defer JS, use CDN |
| Non-indexable page | High | Check robots meta and noindex tags |
Cost Control Tips
- Set
max_crawl_pagesto 100 for initial audits — increase for large sites - Disable
enable_javascriptunless the site is SPA/React — it costs significantly more - Disable
check_spell— rarely actionable and adds cost - Run crawls weekly or monthly — not daily