version: 2
title: Wikipedia non-bot page update event stream
contributor: https://github.com/maglietti
summary: Stream page-update events that were not created by bots
description: |-
  This recipe will separate human generated events from bot generated
  events in the english wikipedia database page-update event stream
  and store them for additional processing.
  API Reference: https://stream.wikimedia.org/?doc#/streams/get_v2_stream_mediawiki_revision_create

ingestStreams:
  - name: wikipedia-revision-create
    source:
      type: ServerSentEvent
      url: https://stream.wikimedia.org/v2/stream/mediawiki.revision-create
      format:
        type: Json
    query: |-
      MATCH (revNode),(pageNode),(dbNode),(userNode)
      WHERE id(revNode) = idFrom('revision', $that.rev_id)
        AND id(pageNode) = idFrom('page', $that.page_id)
        AND id(dbNode) = idFrom('db', $that.database)
        AND id(userNode) = idFrom('id', $that.performer.user_id)

      SET revNode = $that,
          revNode.bot = $that.performer.user_is_bot,
          revNode:revision

      SET pageNode.id = $that.page_id,
          pageNode.namespace = $that.page_namespace,
          pageNode.title = $that.page_title,
          pageNode.comment = $that.comment,
          pageNode.is_redirect = $that.page_is_redirect,
          pageNode:page

      SET dbNode.database = $that.database,
          dbNode:db

      SET userNode = $that.performer,
          userNode.name = $that.performer.user_text,
          userNode:user

      CREATE (revNode)-[:TO]->(pageNode),
             (pageNode)-[:IN]->(dbNode),
             (userNode)-[:RESPONSIBLE_FOR]->(revNode)

      WITH revNode
      WHERE $that.rev_parent_id IS NOT NULL
      MATCH (parentNode)
      WHERE id(parentNode) = idFrom('revision', $that.rev_parent_id)
      SET parentNode.rev_id = $that.rev_parent_id
      CREATE (parentNode)-[:NEXT]->(revNode)

standingQueries:
  - name: not-a-bot
    pattern:
      query: |-
        MATCH (userNode:user {user_is_bot: false})-[:RESPONSIBLE_FOR]->(revNode:revision {database: 'enwiki'})
        RETURN DISTINCT id(revNode) as id
      type: Cypher
    outputs:
      - name: print-output
        resultEnrichment:
          query: |-
            MATCH (n)
            WHERE id(n) = $that.data.id
            RETURN properties(n)
          parameter: that
        destinations:
          - type: StandardOut

nodeAppearances: []
quickQueries: []
sampleQueries: []