Sıfırdan SEO Crawler Yapın: Python ile Adım Adım Rehber

pip install requests beautifulsoup4 lxml

requests==2.31.0
beautifulsoup4==4.12.3
lxml==5.1.0

import requests
from typing import Optional


def fetch_page(url: str, timeout: int = 10) -> Optional[requests.Response]:
    """Verilen URL'ye GET istegi gonderir.

    Args:
        url: Cekilecek sayfa adresi.
        timeout: Baglanti zaman asimi (saniye).

    Returns:
        Basarili ise Response nesnesi, hata durumunda None.
    """
    headers = {
        "User-Agent": "MiniSEOCrawler/1.0 (+https://siteadresiniz.com/bot)"
    }
    try:
        response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
        return response
    except requests.RequestException as e:
        print(f"[HATA] {url}: {e}")
        return None

from bs4 import BeautifulSoup
from dataclasses import dataclass, field
from typing import Dict, List


@dataclass
class SEOData:
    """Bir sayfanin SEO verilerini tutar."""
    url: str
    status_code: int
    title: str = ""
    meta_description: str = ""
    h1_tags: List[str] = field(default_factory=list)
    h2_tags: List[str] = field(default_factory=list)
    internal_links: List[str] = field(default_factory=list)
    external_links: List[str] = field(default_factory=list)
    images_without_alt: int = 0
    word_count: int = 0


def extract_seo_data(url: str, response: requests.Response, base_domain: str) -> SEOData:
    """HTTP yanitindan SEO verilerini cikarir.

    Args:
        url: Sayfanin URL'si.
        response: requests.Response nesnesi.
        base_domain: Site alan adi (ornek: example.com).

    Returns:
        Sayfanin SEO verilerini iceren SEOData nesnesi.
    """
    soup = BeautifulSoup(response.text, "lxml")
    data = SEOData(url=url, status_code=response.status_code)

    # Title etiketi
    title_tag = soup.find("title")
    if title_tag:
        data.title = title_tag.get_text(strip=True)

    # Meta description
    meta_desc = soup.find("meta", attrs={"name": "description"})
    if meta_desc:
        data.meta_description = meta_desc.get("content", "")

    # Baslik etiketleri
    data.h1_tags = [h.get_text(strip=True) for h in soup.find_all("h1")]
    data.h2_tags = [h.get_text(strip=True) for h in soup.find_all("h2")]

    # Baglantilari siniflandir
    for link in soup.find_all("a", href=True):
        href = link["href"]
        if href.startswith(("mailto:", "tel:", "javascript:", "#")):
            continue
        if base_domain in href or href.startswith("/"):
            data.internal_links.append(href)
        elif href.startswith("http"):
            data.external_links.append(href)

    # Alt etiketi olmayan gorseller
    images = soup.find_all("img")
    data.images_without_alt = sum(
        1 for img in images if not img.get("alt", "").strip()
    )

    # Kelime sayisi
    body = soup.find("body")
    if body:
        data.word_count = len(body.get_text(separator=" ", strip=True).split())

    return data

from typing import Any


def extract_meta_tags(soup: BeautifulSoup) -> Dict[str, Any]:
    """Sayfadaki SEO ile ilgili meta etiketlerini cikarir.

    Args:
        soup: BeautifulSoup nesnesi.

    Returns:
        Meta etiketlerinin anahtar-deger eslesmesi.
    """
    meta = {}

    # robots etiketi
    robots_tag = soup.find("meta", attrs={"name": "robots"})
    if robots_tag:
        meta["robots"] = robots_tag.get("content", "")

    # canonical URL
    canonical = soup.find("link", attrs={"rel": "canonical"})
    if canonical:
        meta["canonical"] = canonical.get("href", "")

    # Open Graph etiketleri
    og_tags = soup.find_all("meta", attrs={"property": lambda x: x and x.startswith("og:")})
    for tag in og_tags:
        meta[tag["property"]] = tag.get("content", "")

    # hreflang etiketleri
    hreflangs = soup.find_all("link", attrs={"rel": "alternate", "hreflang": True})
    meta["hreflang"] = [
        {"lang": tag["hreflang"], "href": tag.get("href", "")}
        for tag in hreflangs
    ]

    return meta

import time
from dataclasses import dataclass


@dataclass
class LinkCheckResult:
    """Bir baglanti kontrolunun sonucunu tutar."""
    url: str
    status_code: int
    response_time: float
    redirect_chain: List[str]
    is_broken: bool


def check_links(urls: List[str], timeout: int = 10) -> List[LinkCheckResult]:
    """Verilen URL listesinin erisim durumunu kontrol eder.

    Args:
        urls: Kontrol edilecek URL listesi.
        timeout: Her istek icin zaman asimi (saniye).

    Returns:
        Her URL icin LinkCheckResult listesi.
    """
    results = []
    headers = {
        "User-Agent": "MiniSEOCrawler/1.0 (+https://siteadresiniz.com/bot)"
    }

    for url in urls:
        start = time.time()
        try:
            resp = requests.get(
                url, headers=headers, timeout=timeout, allow_redirects=True
            )
            elapsed = time.time() - start

            # Yonlendirme zincirini cikar
            chain = [r.url for r in resp.history] if resp.history else []

            results.append(LinkCheckResult(
                url=url,
                status_code=resp.status_code,
                response_time=round(elapsed, 2),
                redirect_chain=chain,
                is_broken=resp.status_code >= 400,
            ))
        except requests.RequestException:
            elapsed = time.time() - start
            results.append(LinkCheckResult(
                url=url,
                status_code=0,
                response_time=round(elapsed, 2),
                redirect_chain=[],
                is_broken=True,
            ))

        # Sunucuyu bunaltmamak icin kisa bekleme
        time.sleep(0.5)

    return results

from urllib.parse import urljoin, urlparse
from collections import deque
import xml.etree.ElementTree as ET
from datetime import datetime


def crawl_site(
    start_url: str,
    max_pages: int = 100,
    delay: float = 0.5,
) -> List[SEOData]:
    """Bir siteyi baslangic URL'sinden itibaren tarar.

    Args:
        start_url: Taramin baslayacagi URL.
        max_pages: Taranacak maksimum sayfa sayisi.
        delay: Istekler arasi bekleme suresi (saniye).

    Returns:
        Taranan her sayfa icin SEOData listesi.
    """
    parsed_start = urlparse(start_url)
    base_domain = parsed_start.netloc

    visited: set = set()
    queue: deque = deque([start_url])
    results: List[SEOData] = []

    print(f"[BASLANGIC] {start_url} taranacak (maks {max_pages} sayfa)")

    while queue and len(visited) < max_pages:
        current_url = queue.popleft()

        # Normalize et
        current_url = current_url.split("#")[0]  # Fragment'i at
        if current_url in visited:
            continue

        response = fetch_page(current_url)
        if response is None:
            continue

        visited.add(current_url)
        content_type = response.headers.get("Content-Type", "")

        # Sadece HTML sayfalari isle
        if "text/html" not in content_type:
            continue

        seo_data = extract_seo_data(current_url, response, base_domain)
        results.append(seo_data)

        print(f"  [{len(visited)}/{max_pages}] {response.status_code} — {current_url}")

        # Dahili baglantilari kuyruga ekle
        for link in seo_data.internal_links:
            full_url = urljoin(current_url, link)
            full_url = full_url.split("#")[0]
            parsed = urlparse(full_url)

            if parsed.netloc == base_domain and full_url not in visited:
                queue.append(full_url)

        time.sleep(delay)

    print(f"[BITTI] {len(results)} sayfa tarandi.")
    return results


def generate_sitemap_xml(pages: List[SEOData], output_path: str = "sitemap.xml") -> None:
    """Taranan sayfalardan sitemap.xml olusturur.

    Args:
        pages: SEOData listesi.
        output_path: Cikti dosya yolu.
    """
    urlset = ET.Element("urlset")
    urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")

    for page in pages:
        if page.status_code != 200:
            continue
        url_el = ET.SubElement(urlset, "url")
        loc = ET.SubElement(url_el, "loc")
        loc.text = page.url
        lastmod = ET.SubElement(url_el, "lastmod")
        lastmod.text = datetime.now().strftime("%Y-%m-%d")

    tree = ET.ElementTree(urlset)
    ET.indent(tree, space="  ")
    tree.write(output_path, encoding="unicode", xml_declaration=True)
    print(f"[SITEMAP] {output_path} olusturuldu ({len(pages)} URL)")

import csv
import json


def export_to_csv(pages: List[SEOData], output_path: str = "seo_report.csv") -> None:
    """SEO verilerini CSV formatinda kaydeder.

    Args:
        pages: SEOData listesi.
        output_path: Cikti dosya yolu.
    """
    fieldnames = [
        "url", "status_code", "title", "meta_description",
        "h1_count", "h2_count", "internal_links", "external_links",
        "images_without_alt", "word_count",
    ]

    with open(output_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for page in pages:
            writer.writerow({
                "url": page.url,
                "status_code": page.status_code,
                "title": page.title,
                "meta_description": page.meta_description,
                "h1_count": len(page.h1_tags),
                "h2_count": len(page.h2_tags),
                "internal_links": len(page.internal_links),
                "external_links": len(page.external_links),
                "images_without_alt": page.images_without_alt,
                "word_count": page.word_count,
            })

    print(f"[CSV] {output_path} kaydedildi ({len(pages)} satir)")


def export_to_json(pages: List[SEOData], output_path: str = "seo_report.json") -> None:
    """SEO verilerini JSON formatinda kaydeder.

    Args:
        pages: SEOData listesi.
        output_path: Cikti dosya yolu.
    """
    from dataclasses import asdict

    data = [asdict(page) for page in pages]
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)

    print(f"[JSON] {output_path} kaydedildi ({len(pages)} kayit)")

"""
MiniSEOCrawler — Basit bir SEO site tarayicisi.

Kullanim:
    python crawler.py https://example.com --max-pages 50
"""

import argparse


def main() -> None:
    """CLI argumanlariyla crawler'i baslatir."""
    parser = argparse.ArgumentParser(description="MiniSEOCrawler — SEO site tarayicisi")
    parser.add_argument("url", help="Taranacak sitenin baslangic URL'si")
    parser.add_argument("--max-pages", type=int, default=50, help="Maks sayfa sayisi (varsayilan: 50)")
    parser.add_argument("--delay", type=float, default=0.5, help="Istekler arasi bekleme (saniye)")
    parser.add_argument("--output", default="seo_report", help="Cikti dosya adi (uzantisiz)")
    args = parser.parse_args()

    # Siteyi tara
    pages = crawl_site(args.url, max_pages=args.max_pages, delay=args.delay)

    if not pages:
        print("Hicbir sayfa tarananamadi.")
        return

    # Kirik baglantilari kontrol et
    all_links = set()
    for page in pages:
        for link in page.internal_links:
            full = urljoin(page.url, link)
            all_links.add(full)

    print(f"\n[LINK KONTROL] {len(all_links)} benzersiz dahili baglanti kontrol ediliyor...")
    broken = [r for r in check_links(list(all_links)[:200]) if r.is_broken]

    if broken:
        print(f"\n  {len(broken)} kirik baglanti bulundu:")
        for b in broken:
            print(f"    {b.status_code} — {b.url}")
    else:
        print("  Kirik baglanti yok.")

    # Sonuclari kaydet
    export_to_csv(pages, f"{args.output}.csv")
    export_to_json(pages, f"{args.output}.json")
    generate_sitemap_xml(pages, f"{args.output}_sitemap.xml")

    # Ozet rapor
    print(f"\n{'='*50}")
    print(f"  TARAMA OZETI")
    print(f"{'='*50}")
    print(f"  Taranan sayfa:          {len(pages)}")
    print(f"  200 OK:                 {sum(1 for p in pages if p.status_code == 200)}")
    print(f"  Yonlendirme (3xx):      {sum(1 for p in pages if 300 <= p.status_code < 400)}")
    print(f"  Hata (4xx/5xx):         {sum(1 for p in pages if p.status_code >= 400)}")
    print(f"  Alt etiketsiz gorsel:   {sum(p.images_without_alt for p in pages)}")
    print(f"  Kirik baglanti:         {len(broken)}")
    print(f"{'='*50}")


if __name__ == "__main__":
    main()

python crawler.py https://siteadresiniz.com --max-pages 100 --delay 1