/* See LICENSE file for copyright and license details. */ #ifndef ROBOTS_H#define ROBOTS_H /* Maximum rules per robots.txt */#define MAX_RULES 512 /* Rule types */typedef struct {	char *path;	int allow; /* 1 = allow, 0 = disallow */} RobotsRule; /* Parsed robots.txt for a domain */typedef struct {	char *domain;	RobotsRule rules[MAX_RULES];	int nrules;	int crawl_delay; /* seconds, 0 = none specified */} Robots; /* Fetch and parse robots.txt for a domain */Robots *robots_fetch(const char *domain); /* Check if a path is allowed */int robots_allowed(Robots *r, const char *path); /* Get crawl delay in seconds (0 = none) */int robots_delay(Robots *r); /* Free robots struct */void robots_free(Robots *r); #endif /* ROBOTS_H */