marketingskills/skills/competitor-profiling/evals/evals.json

{
  "skill_name": "competitor-profiling",
  "evals": [
    {
      "id": 1,
      "prompt": "Profile these three competitors for us: https://competitor1.com, https://competitor2.com, https://competitor3.com. We need this for sales enablement and to find positioning gaps.",
      "expected_output": "Should check for product-marketing.md first. Should run the full research process: Phase 1 site scraping (Firecrawl map + scrape of homepage, pricing, features, about, customers, integrations, changelog), Phase 2 SEO and market data (DataForSEO for backlinks, ranked keywords, traffic, competitors), Phase 3 synthesis. Should save raw data to competitor-profiles/raw/<slug>/<YYYY-MM-DD>/ with scrapes/, seo/, reviews/ subfolders before synthesizing. Should produce one markdown file per competitor following the profile template (At a Glance, Positioning & Messaging, Product & Features, Pricing, Customers & Social Proof, SEO & Content Strategy, Strengths & Weaknesses, Competitive Implications). Should produce a _summary.md after individual profiles with comparison table, positioning map, key takeaways, gaps and opportunities. Should parallelize scraping when handling multiple competitors and use consistent metrics across all three for comparability.",
      "assertions": [
        "Checks for product-marketing.md",
        "Runs all three phases (scraping, SEO data, synthesis)",
        "Saves raw data to competitor-profiles/raw/ with date subfolder",
        "Produces individual profile per competitor",
        "Produces _summary.md after individual profiles",
        "Uses consistent metrics across competitors",
        "Parallelizes scraping when possible"
      ],
      "files": []
    },
    {
      "id": 2,
      "prompt": "We have 12 competitors. Profile all of them.",
      "expected_output": "Should recommend prioritizing rather than profiling all 12. Should suggest profiling the top 5 first based on domain overlap or market similarity (handling-multiple-competitors guidance). Should default to quick scan mode for a list this size, not deep profile. Should explain the difference: quick scan covers homepage + pricing + domain rank overview + ranked keywords summary, deep profile adds reviews, technology stack, backlink details. Should offer deep profile only if user requests or for 3 or fewer competitors. Should ask which competitors are highest priority if user wants to narrow further.",
      "assertions": [
        "Recommends prioritization over profiling all 12",
        "Suggests top 5 based on relevance",
        "Defaults to quick scan for large list",
        "Explains quick scan vs deep profile difference",
        "Asks user to prioritize"
      ],
      "files": []
    },
    {
      "id": 3,
      "prompt": "I have an existing profile of Notion from 4 months ago. Should I update it or start fresh?",
      "expected_output": "Should explain profile updating process from the Updating Profiles section. Should recommend updating rather than starting fresh — preserves history and enables diffing. Should explain what to re-pull: pricing page first (most volatile), SEO metrics (traffic and rankings shift monthly), changelog scan for product changes. Should update the Generated date. Should add a Change Log section at the bottom noting what changed since last profile. Should also save the new raw data to a new <YYYY-MM-DD> folder rather than overwriting prior data — supports diffing over time.",
      "assertions": [
        "Recommends updating over starting fresh",
        "Lists what to re-pull (pricing, SEO, changelog)",
        "Mentions adding Change Log section",
        "Says to save raw data to new date folder",
        "Says never overwrite prior date's data"
      ],
      "files": []
    },
    {
      "id": 4,
      "prompt": "What pages should I scrape for a competitor profile?",
      "expected_output": "Should list the prioritized page types from Phase 1: homepage, pricing page, features/product pages, about/company page, blog (top-level for content strategy signals), customers/case studies page, integrations page, changelog/what's new (if exists). Should explain what to extract from each: homepage (headline, value prop, primary CTA, social proof, target audience signals), pricing (tiers, prices, feature breakdown, billing options, free tier/trial details), features (categories, key capabilities, how they describe each feature), about (founding story, team size, funding, mission, HQ), customers (named customers, logos, industries, case study themes), integrations (count, key integrations, categories), changelog (release velocity, recent focus areas, product direction signals). Should mention optional review scraping (G2, Capterra, Product Hunt, TrustRadius).",
      "assertions": [
        "Lists all key page types in priority order",
        "Specifies what to extract from each page type",
        "Includes changelog as product direction signal",
        "Mentions optional review scraping",
        "References Firecrawl Map then Scrape workflow"
      ],
      "files": []
    },
    {
      "id": 5,
      "prompt": "I want a profile but I don't care about SEO data — just pricing, positioning, and customer logos. Can you skip the DataForSEO calls?",
      "expected_output": "Should accept the scoped request and skip Phase 2. Should run Phase 1 (Firecrawl scraping of homepage, pricing, customers pages) and Phase 3 synthesis only. Should explain that without SEO data, the profile won't include Domain Rank, organic traffic estimates, ranked keywords, referring domains, or top organic pages — but the positioning, pricing, and customer sections will be complete. Should produce an abbreviated profile flagging the SEO section as 'not collected per user request' rather than leaving placeholders. Should still save raw scrapes to disk for reuse.",
      "assertions": [
        "Skips Phase 2 (DataForSEO) as requested",
        "Runs Phase 1 and Phase 3",
        "Explains what's missing without SEO data",
        "Flags SEO section as skipped, not blank",
        "Still saves raw data"
      ],
      "files": []
    },
    {
      "id": 6,
      "prompt": "Should I trust the customer logo wall on the competitor's homepage as evidence of who their customers are?",
      "expected_output": "Should apply the 'Facts Over Opinions' and 'Honest Assessment' principles. Should explain that customer logos are a positioning claim, not necessarily an accurate customer breakdown — companies often show their best-known logos regardless of share of revenue. Should recommend cross-referencing: check case studies for actual usage details, search for press releases naming customers, look at customer reviews on G2/Capterra/TrustRadius for company name signals, check their LinkedIn for posts about customers. Should note: if they claim '10,000 customers' but have weak traffic/backlink profile, the claim should be flagged in the profile. Should distinguish between named customers (verifiable claims) and 'industries served' (positioning statement). Always include the date the data was pulled.",
      "assertions": [
        "Treats logos as positioning claim, not customer breakdown",
        "Recommends cross-referencing case studies and reviews",
        "Mentions checking traffic/backlink profile against claim scale",
        "Distinguishes verifiable named customers from claims",
        "Notes including date pulled"
      ],
      "files": []
    }
  ]
}