sfp_spider.py

-*- coding: utf-8 -*- ------------------------------------------------------------------------------- Name: sfp_spider Purpose: SpiderFoot plug-in for spidering sites and returning meta data for other plug-ins to consume. Author: Steve Micallef <steve@binarypool.com>

# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_spider
# Purpose:      SpiderFoot plug-in for spidering sites and returning meta data
#               for other plug-ins to consume.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     25/03/2012
# Copyright:   (c) Steve Micallef 2012
# Licence:     MIT
# -------------------------------------------------------------------------------

import json
import time

from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin


class sfp_spider(SpiderFootPlugin):

    meta = {
        'name': "Web Spider",
        'summary': "Spidering of web-pages to extract content for searching.",
        'flags': ["slow"],
        'useCases': ["Footpr

... (truncated -- full source via MCP)