physicsforums.go

// Package physicsforums is the library behind the physicsforums command line:
// the HTTP client, request shaping, and the typed data models for PhysicsForums.com.
//
// The Client here is the spine every command shares. It sets a real
// User-Agent, paces requests so a busy session stays polite, and retries the
// transient failures (429 and 5xx) that any public site throws under load.
package physicsforums

import (
	"context"
	"fmt"
	"html"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

const DefaultUserAgent = "Mozilla/5.0 (compatible; physicsforums-cli/0.1; +https://github.com/tamnd/physicsforums-cli)"

// Config holds constructor parameters for the client.
type Config struct {
	BaseURL   string
	UserAgent string
	Rate      time.Duration
	Retries   int
	Timeout   time.Duration
}

// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
	return Config{
		BaseURL:   "https://www.physicsforums.com",
		UserAgent: DefaultUserAgent,
		Rate:      500 * time.Millisecond,
		Retries:   3,
		Timeout:   30 * time.Second,
	}
}

// Thread represents a single discussion thread on PhysicsForums.
type Thread struct {
	Rank    int    `json:"rank"`
	ID      int    `json:"id"`
	Title   string `json:"title"`
	Forum   string `json:"forum"`
	Date    string `json:"date"`
	Replies int    `json:"replies"`
	Views   int    `json:"views"`
	URL     string `json:"url"`
}

// Forum represents a known PhysicsForums.com forum section.
type Forum struct {
	Rank int    `json:"rank"`
	ID   int    `json:"id"`
	Slug string `json:"slug"`
	Name string `json:"name"`
}

// KnownForums is the hardcoded list of PhysicsForums.com forum sections.
var KnownForums = []Forum{
	{1, 61, "classical-physics", "Classical Physics"},
	{2, 62, "quantum-physics", "Quantum Physics"},
	{3, 70, "special-and-general-relativity", "Special and General Relativity"},
	{4, 65, "high-energy-nuclear-particle-physics", "High Energy, Nuclear, Particle Physics"},
	{5, 64, "atomic-and-condensed-matter", "Atomic and Condensed Matter"},
	{6, 66, "beyond-the-standard-models", "Beyond the Standard Models"},
	{7, 71, "astronomy-and-astrophysics", "Astronomy and Astrophysics"},
	{8, 69, "cosmology", "Cosmology"},
	{9, 111, "other-physics-topics", "Other Physics Topics"},
	{10, 73, "general-math", "General Math"},
	{11, 109, "calculus", "Calculus"},
	{12, 228, "topology-and-analysis", "Topology and Analysis"},
	{13, 75, "linear-and-abstract-algebra", "Linear and Abstract Algebra"},
	{14, 76, "differential-geometry", "Differential Geometry"},
	{15, 83, "chemistry", "Chemistry"},
	{16, 165, "programming-and-computer-science", "Programming and Computer Science"},
	{17, 84, "earth-sciences", "Earth Sciences"},
	{18, 292, "quantum-interpretations-and-foundations", "Quantum Interpretations and Foundations"},
}

// Forums returns the list of known PhysicsForums.com forum sections.
func Forums() []Forum { return KnownForums }

// regex patterns for parsing XenForo thread HTML
var (
	splitRe    = regexp.MustCompile(`class="structItem structItem--thread`)
	titleURLRe = regexp.MustCompile(`href="(/threads/([^"]+)\.(\d+)/)"[^>]*data-tp-primary="on"[^>]*>([\s\S]*?)</a>`)
	dateRe     = regexp.MustCompile(`datetime="(\d{4}-\d{2}-\d{2})`)
	repliesRe  = regexp.MustCompile(`<dt>Replies</dt>\s*<dd>(\d+)</dd>`)
	viewsRe    = regexp.MustCompile(`<dt>Views</dt>\s*<dd>([\d,]+)</dd>`)
	tagRe      = regexp.MustCompile(`<[^>]+>`)
)

func parseThreads(raw, forum string, limit int, baseURL string) []Thread {
	parts := splitRe.Split(raw, -1)
	if len(parts) <= 1 {
		return nil
	}
	var out []Thread
	rank := 0
	for _, part := range parts[1:] {
		m := titleURLRe.FindStringSubmatch(part)
		if m == nil {
			continue
		}
		path := m[1]
		idStr := m[3]
		id, _ := strconv.Atoi(idStr)
		rawTitle := strings.TrimSpace(m[4])
		title := tagRe.ReplaceAllString(rawTitle, "")
		title = html.UnescapeString(strings.TrimSpace(title))
		if title == "" {
			continue
		}

		date := ""
		if dm := dateRe.FindStringSubmatch(part); dm != nil {
			date = dm[1]
		}

		replies := 0
		if rm := repliesRe.FindStringSubmatch(part); rm != nil {
			replies, _ = strconv.Atoi(rm[1])
		}

		views := 0
		if vm := viewsRe.FindStringSubmatch(part); vm != nil {
			vStr := strings.ReplaceAll(vm[1], ",", "")
			views, _ = strconv.Atoi(vStr)
		}

		rank++
		if limit > 0 && rank > limit {
			break
		}
		out = append(out, Thread{
			Rank:    rank,
			ID:      id,
			Title:   title,
			Forum:   forum,
			Date:    date,
			Replies: replies,
			Views:   views,
			URL:     baseURL + path,
		})
	}
	return out
}

// Client talks to PhysicsForums.com over HTTP.
type Client struct {
	cfg        Config
	httpClient *http.Client
	mu         sync.Mutex
	last       time.Time
}

// NewClient returns a Client with the given config.
func NewClient(cfg Config) *Client {
	return &Client{
		cfg:        cfg,
		httpClient: &http.Client{Timeout: cfg.Timeout},
	}
}

// List fetches recent threads from the given forum slug.
func (c *Client) List(ctx context.Context, forum string, limit int) ([]Thread, error) {
	rawURL := fmt.Sprintf("%s/forums/%s/", c.cfg.BaseURL, forum)
	raw, err := c.get(ctx, rawURL)
	if err != nil {
		return nil, err
	}
	threads := parseThreads(string(raw), forum, limit, c.cfg.BaseURL)
	return threads, nil
}

func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) {
	var lastErr error
	for attempt := 0; attempt <= c.cfg.Retries; attempt++ {
		if attempt > 0 {
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(backoff(attempt)):
			}
		}
		b, retry, err := c.do(ctx, rawURL)
		if err == nil {
			return b, nil
		}
		lastErr = err
		if !retry {
			return nil, err
		}
	}
	return nil, fmt.Errorf("get: %w", lastErr)
}

func (c *Client) do(ctx context.Context, rawURL string) ([]byte, bool, error) {
	c.pace()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
	if err != nil {
		return nil, false, err
	}
	req.Header.Set("User-Agent", c.cfg.UserAgent)

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, true, err
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
		return nil, true, fmt.Errorf("http %d", resp.StatusCode)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, false, fmt.Errorf("http %d", resp.StatusCode)
	}

	b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	if err != nil {
		return nil, true, err
	}
	return b, false, nil
}

func (c *Client) pace() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.cfg.Rate <= 0 {
		return
	}
	if wait := c.cfg.Rate - time.Since(c.last); wait > 0 {
		time.Sleep(wait)
	}
	c.last = time.Now()
}

func backoff(attempt int) time.Duration {
	d := time.Duration(attempt) * 500 * time.Millisecond
	if d > 5*time.Second {
		d = 5 * time.Second
	}
	return d
}