Bright Data: Dataset Marketplaceでデータをダウンロード・分析する

!pip install requests pandas python-dotenv matplotlib -q

import os
from dotenv import load_dotenv
import requests
import pandas as pd
import csv
import json

# Load environment variables from .env file
load_dotenv()

# Get the API key (expects BRIGHTDATA_API_KEY in your .env file)
api_key = os.getenv("BRIGHT_DATA_API_KEY")

if not api_key:
    raise ValueError("Missing BRIGHTDATA_API_KEY environment variable. Make sure it is set in your .env file.")

import csv
import requests
import pandas as pd

SNAPSHOT_ID = "snap_mewa1lct2l63mtpvge"
URL = f"https://api.brightdata.com/datasets/snapshots/{SNAPSHOT_ID}/download?format=csv"

if not api_key:
    raise ValueError("Missing BRIGHT_DATA_API_KEY (ensure it's set and loaded before running).")

response = requests.get(
    URL,
    headers={"Authorization": f"Bearer {api_key}"},
    stream=True,
    timeout=120,
)

if response.status_code != 200:
    print("HTTP", response.status_code, response.text[:300])
else:
    line_iter = response.iter_lines(decode_unicode=True)

    header_line = next(line_iter, None)
    if not header_line:
        raise ValueError("Empty snapshot (no header line received).")

    header = next(csv.reader([header_line]))

    try:
        title_idx = header.index("title")
    except ValueError:
        title_idx = None  # proceed without title filtering if column absent

    kept_rows = []
    for line in line_iter:
        if len(kept_rows) >= 100:
            break
        if not line.strip():
            continue
        row = next(csv.reader([line]))

        if title_idx is not None:
            title_val = row[title_idx].strip().lower()
            if title_val in ("", "null", "none", "nan"):
                continue

        kept_rows.append(row)

    df_marketplace = pd.DataFrame(kept_rows, columns=header)
    print(f"Rows kept: {len(df_marketplace)}")
    display(df_marketplace.head())

# Basic cleaning & exploratory analysis
import json
import matplotlib.pyplot as plt

if 'df_marketplace' in globals() and not df_marketplace.empty:
    # Convert rating/count columns to numeric
    df_marketplace['imdb_rating'] = pd.to_numeric(df_marketplace.get('imdb_rating'), errors='coerce')
    df_marketplace['imdb_rating_count'] = pd.to_numeric(df_marketplace.get('imdb_rating_count'), errors='coerce')

    # Drop rows missing essentials
    df_cleaned = df_marketplace.dropna(subset=['imdb_rating', 'title', 'genres'])
    df_cleaned = df_cleaned[df_cleaned['title'].str.strip() != '']

    print(f"Cleaned data rows: {len(df_cleaned)}")

    # --- Analysis 1: Most Common Genres ---
    def parse_genres(genres_str):
        try:
            return json.loads(genres_str.replace("'", '"'))
        except (json.JSONDecodeError, AttributeError):
            return []

    all_genres = df_cleaned['genres'].apply(parse_genres).explode()
    genre_counts = all_genres.value_counts()

    print("\n--- Top 10 Genres ---")
    print(genre_counts.head(10))

    # --- Analysis 2: Highest-Rated Movies (with >10k ratings) ---
    popular_movies = df_cleaned[df_cleaned['imdb_rating_count'] > 10000]
    top_movies = popular_movies.sort_values(by='imdb_rating', ascending=False)

    print("\n--- Top 10 Highest-Rated Popular Movies ---")
    print(top_movies[['title', 'imdb_rating', 'imdb_rating_count']].head(10))

    # --- Analysis 3: Rating Distribution ---
    plt.figure(figsize=(10, 6))
    plt.hist(df_cleaned['imdb_rating'], bins=20, color='skyblue', edgecolor='black')
    plt.title('Distribution of IMDB Ratings')
    plt.xlabel('IMDB Rating')
    plt.ylabel('Number of Movies')
    plt.grid(axis='y', alpha=0.75)
    plt.show()
else:
    print("DataFrame 'df_marketplace' is empty or not defined.")