Appearance
Workflow: Competitor Analysis Fresh
This workflow starts from a list of target keywords, extracts competing domains from SERP results, then compares backlink authority and keyword overlap against your client's domain.
Workflow Sequence
sequenceDiagram
participant Y as Your Script
participant SERP as SERP API
participant BL as Backlinks API
participant Labs as Labs API
Y->>SERP: POST tasks_post (target keywords)
SERP-->>Y: task IDs
Y->>SERP: GET tasks_ready (poll)
SERP-->>Y: results ready
Y->>SERP: GET task_get for each keyword
SERP-->>Y: top 10 organic results per keyword
Note over Y: Extract unique competing domains from results
Y->>BL: POST domain summary for each competitor
BL-->>Y: referring domains, authority metrics
Y->>BL: POST domain summary for client
BL-->>Y: client metrics
Note over Y: Build comparison matrix
Y->>Labs: POST competitors_domain for client
Labs-->>Y: competitor overlap data
Y->>Labs: POST keyword_gap for client vs each competitor
Labs-->>Y: keywords competitor ranks for that client doesn't
Note over Y: Compile final comparison reportStep 1: Identify Competitors via SERP
Pull SERP results for your target keywords and extract the domains that appear most often.
python
import os
import time
import json
import requests
from collections import Counter
from requests.auth import HTTPBasicAuth
from urllib.parse import urlparse
auth = HTTPBasicAuth(
os.environ["DATAFORSEO_LOGIN"],
os.environ["DATAFORSEO_PASSWORD"]
)
CLIENT_DOMAIN = "clientsite.com"
TARGET_KEYWORDS = [
"roofing contractor miami",
"roof repair miami",
"metal roofing miami",
"flat roof miami",
"commercial roofing miami",
"residential roofing miami"
]
# Post SERP tasks
serp_payload = [
{
"keyword": kw,
"location_code": 1015116, # Miami, FL
"language_code": "en",
"device": "desktop",
"depth": 10 # top 10 is enough for competitor identification
}
for kw in TARGET_KEYWORDS
]
response = requests.post(
"https://api.dataforseo.com/v3/serp/google/organic/tasks_post",
auth=auth,
json=serp_payload
)
task_ids = [t["id"] for t in response.json()["tasks"] if t["status_code"] == 20100]
print(f"Posted {len(task_ids)} SERP tasks")
# Poll for results
def collect_serp_results_simple(auth, task_ids, max_wait=180):
ready = set()
results = {}
deadline = time.time() + max_wait
while time.time() < deadline and len(ready) < len(task_ids):
data = requests.get(
"https://api.dataforseo.com/v3/serp/google/organic/tasks_ready",
auth=auth
).json()
for task in data["tasks"]:
for r in task.get("result", []):
if r["id"] in task_ids:
ready.add(r["id"])
if len(ready) < len(task_ids):
time.sleep(20)
for tid in ready:
data = requests.get(
f"https://api.dataforseo.com/v3/serp/google/organic/task_get/advanced/{tid}",
auth=auth
).json()
task = data["tasks"][0]
keyword = task["data"]["keyword"]
items = task["result"][0].get("items", []) if task.get("result") else []
results[keyword] = items
return results
serp_results = collect_serp_results_simple(auth, set(task_ids))Step 2: Extract Competing Domains
Count how often each domain appears in the top 10 results across all keywords.
python
def extract_competitors(serp_results, client_domain, top_n=10):
"""Find domains that appear most in SERP results."""
domain_appearances = Counter()
domain_keywords = {}
for keyword, items in serp_results.items():
for item in items:
if item.get("type") != "organic":
continue
url = item.get("url", "")
if not url:
continue
domain = urlparse(url).netloc.replace("www.", "")
if domain == client_domain:
continue # skip the client
domain_appearances[domain] += 1
if domain not in domain_keywords:
domain_keywords[domain] = []
domain_keywords[domain].append(keyword)
# Top N competitors by keyword overlap
top_competitors = [
{
"domain": domain,
"keyword_overlap": count,
"shared_keywords": domain_keywords[domain]
}
for domain, count in domain_appearances.most_common(top_n)
]
return top_competitors
competitors = extract_competitors(serp_results, CLIENT_DOMAIN, top_n=8)
print(f"\nTop Competitors by SERP Overlap:")
print(f"{'Domain':<40} {'Keywords':>8}")
print("-" * 50)
for c in competitors:
print(f"{c['domain']:<40} {c['keyword_overlap']:>8}")Step 3: Compare Backlink Profiles
Pull domain authority metrics for the client and each competitor.
python
def get_domain_authority_metrics(auth, domains):
"""Get backlink summary metrics for multiple domains."""
results = {}
# Batch up to 5 domains per request for efficiency
for domain in domains:
payload = [
{
"target": domain,
"include_subdomains": True
}
]
response = requests.post(
"https://api.dataforseo.com/v3/backlinks/domain_pages_summary/live",
auth=auth,
json=payload
)
data = response.json()
task = data["tasks"][0]
if task.get("result"):
r = task["result"][0]
results[domain] = {
"referring_domains": r.get("referring_domains", 0),
"referring_domains_dofollow": r.get("referring_domains_dofollow", 0),
"backlinks": r.get("backlinks", 0),
"spam_score": r.get("spam_score", 0),
"broken_backlinks": r.get("broken_backlinks", 0)
}
else:
results[domain] = None
time.sleep(0.3) # small delay between requests
return results
# Get metrics for client + all competitors
all_domains = [CLIENT_DOMAIN] + [c["domain"] for c in competitors]
authority_data = get_domain_authority_metrics(auth, all_domains)
# Print comparison matrix
print(f"\nBACKLINK PROFILE COMPARISON")
print(f"{'Domain':<35} {'Ref. Domains':>13} {'Dofollow':>9} {'Spam':>5}")
print("-" * 65)
# Client first
if CLIENT_DOMAIN in authority_data and authority_data[CLIENT_DOMAIN]:
d = authority_data[CLIENT_DOMAIN]
print(f"{'[CLIENT] ' + CLIENT_DOMAIN:<35} {d['referring_domains']:>13,} {d['referring_domains_dofollow']:>9,} {d['spam_score']:>5}")
for c in competitors:
domain = c["domain"]
if domain in authority_data and authority_data[domain]:
d = authority_data[domain]
print(f"{domain:<35} {d['referring_domains']:>13,} {d['referring_domains_dofollow']:>9,} {d['spam_score']:>5}")Step 4: Keyword Gap Analysis via Labs
Find keywords competitors rank for that your client doesn't.
python
def get_keyword_gap(auth, client_domain, competitor_domain, location_code=2840, limit=50):
"""Get keywords competitor ranks for that client doesn't."""
payload = [
{
"target1": competitor_domain,
"target2": client_domain,
"location_code": location_code,
"language_code": "en",
"exclude_top_domains": False,
"filters": [
["keyword_data.keyword_info.search_volume", ">", 100],
["ranked_serp_element.serp_item.rank_absolute", "<=", 20]
],
"order_by": ["keyword_data.keyword_info.search_volume,desc"],
"limit": limit
}
]
response = requests.post(
"https://api.dataforseo.com/v3/dataforseo_labs/google/keywords_for_categories_and_keywords/live",
auth=auth,
json=payload
)
# Note: The exact endpoint may vary — use competitor_keywords endpoint
# Use this endpoint for competitive gap:
payload2 = [
{
"targets": [competitor_domain],
"location_code": location_code,
"language_code": "en",
"filters": [
["keyword_data.keyword_info.search_volume", ">", 100]
],
"order_by": ["keyword_data.keyword_info.search_volume,desc"],
"limit": limit
}
]
response2 = requests.post(
"https://api.dataforseo.com/v3/dataforseo_labs/google/ranked_keywords/live",
auth=auth,
json=payload2
)
data = response2.json()
task = data["tasks"][0]
gap_keywords = []
if task.get("result"):
for item in task["result"][0].get("items", []):
gap_keywords.append({
"keyword": item["keyword_data"]["keyword"],
"volume": item["keyword_data"]["keyword_info"].get("search_volume", 0),
"competitor_position": item.get("ranked_serp_element", {}).get("serp_item", {}).get("rank_absolute")
})
return gap_keywords
# Get top keyword gap for the #1 competitor
if competitors:
top_competitor = competitors[0]["domain"]
gap = get_keyword_gap(auth, CLIENT_DOMAIN, top_competitor)
print(f"\nKEYWORD GAP: {top_competitor} ranks for, {CLIENT_DOMAIN} doesn't")
print(f"{'Keyword':<50} {'Volume':>8} {'Comp. Pos.':>11}")
print("-" * 72)
for kw in gap[:20]:
print(f"{kw['keyword']:<50} {kw['volume']:>8,} {str(kw['competitor_position']):>11}")Step 5: Compile Comparison Report
python
def compile_competitor_report(client_domain, competitors, authority_data, gap_data):
"""Build a structured competitor comparison report."""
report = {
"client": client_domain,
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"competitor_count": len(competitors),
"authority_comparison": {},
"serp_overlap": competitors,
"keyword_gaps": gap_data
}
# Normalize authority data relative to client
client_rd = (authority_data.get(client_domain) or {}).get("referring_domains", 1)
for domain, metrics in authority_data.items():
if not metrics:
continue
report["authority_comparison"][domain] = {
**metrics,
"vs_client_rd_ratio": round(metrics["referring_domains"] / client_rd, 2) if client_rd else 0
}
return report
report = compile_competitor_report(CLIENT_DOMAIN, competitors, authority_data, gap)
# Save report
with open(f"competitor-analysis-{CLIENT_DOMAIN}.json", "w") as f:
json.dump(report, f, indent=2)
print(f"\nCompetitor analysis saved.")
# Print key takeaway
if competitors:
top = competitors[0]
client_rd = (authority_data.get(CLIENT_DOMAIN) or {}).get("referring_domains", 0)
comp_rd = (authority_data.get(top["domain"]) or {}).get("referring_domains", 0)
print(f"\nKEY TAKEAWAY:")
print(f" Top competitor: {top['domain']}")
print(f" Shared keywords: {top['keyword_overlap']}/{len(TARGET_KEYWORDS)}")
print(f" Their referring domains: {comp_rd:,} vs your client: {client_rd:,}")
gap_ratio = comp_rd / client_rd if client_rd else float("inf")
print(f" Backlink gap: {gap_ratio:.1f}x more referring domains")Interpreting Results
| Metric | What It Tells You |
|---|---|
| Keyword overlap count | How often a competitor appears for your target terms — higher = more direct competitor |
| Referring domains gap | Link building gap — how many more links your client needs to be competitive |
| Dofollow ratio | Link quality — a very high ratio may indicate link schemes; very low may indicate heavy nofollow from press |
| Spam score | Domain health — elevated spam score on a competitor can mean an opportunity to compete without risk |
| Keyword gap size | Content gap — keywords your client could target to compete |