Repository: https://github.com/GoogleCloudPlatform/knowledge-catalog
Files analyzed: 230

Directory structure:
└── GoogleCloudPlatform-knowledge-catalog/
    ├── okf
    │   ├── bundles
    │   │   ├── crypto_bitcoin
    │   │   │   ├── datasets
    │   │   │   │   ├── crypto_bitcoin.md
    │   │   │   │   └── index.md
    │   │   │   ├── tables
    │   │   │   │   ├── blocks.md
    │   │   │   │   ├── index.md
    │   │   │   │   ├── inputs.md
    │   │   │   │   ├── outputs.md
    │   │   │   │   └── transactions.md
    │   │   │   ├── index.md
    │   │   │   └── viz.html
    │   │   ├── ga4
    │   │   │   ├── datasets
    │   │   │   │   ├── ga4_obfuscated_sample_ecommerce.md
    │   │   │   │   └── index.md
    │   │   │   ├── references
    │   │   │   │   ├── joins
    │   │   │   │   │   ├── events___ads_clickstats.md
    │   │   │   │   │   └── index.md
    │   │   │   │   ├── metrics
    │   │   │   │   │   ├── avg_pageviews.md
    │   │   │   │   │   ├── avg_spend_per_purchase_session_by_user.md
    │   │   │   │   │   ├── avg_transactions_per_purchaser.md
    │   │   │   │   │   ├── day_count.md
    │   │   │   │   │   ├── event_count.md
    │   │   │   │   │   ├── index.md
    │   │   │   │   │   ├── new_user_count.md
    │   │   │   │   │   ├── overall_avg_spend_per_purchase_session.md
    │   │   │   │   │   └── user_count.md
    │   │   │   │   └── index.md
    │   │   │   ├── tables
    │   │   │   │   ├── events_.md
    │   │   │   │   └── index.md
    │   │   │   ├── index.md
    │   │   │   └── viz.html
    │   │   └── stackoverflow
    │   │       ├── datasets
    │   │       │   ├── index.md
    │   │       │   └── stackoverflow.md
    │   │       ├── references
    │   │       │   ├── badge_classes.md
    │   │       │   ├── close_as_off_topic_reason_types.md
    │   │       │   ├── close_reason_types.md
    │   │       │   ├── creative_commons_by_sa_4_0_license.md
    │   │       │   ├── flag_types.md
    │   │       │   ├── index.md
    │   │       │   ├── link_types.md
    │   │       │   ├── pending_flags.md
    │   │       │   ├── post_feedback.md
    │   │       │   ├── post_history_type_ids.md
    │   │       │   ├── post_notice_class_ids.md
    │   │       │   ├── post_notice_duration_ids.md
    │   │       │   ├── post_notice_type_definitions.md
    │   │       │   ├── post_notice_types.md
    │   │       │   ├── post_notices.md
    │   │       │   ├── post_type_ids.md
    │   │       │   ├── related_questions.md
    │   │       │   ├── review_rejection_reasons.md
    │   │       │   ├── review_task_result_types.md
    │   │       │   ├── review_task_results.md
    │   │       │   ├── review_task_states.md
    │   │       │   ├── review_task_types.md
    │   │       │   ├── review_tasks.md
    │   │       │   ├── sede_databases.md
    │   │       │   ├── sede_databases_markdown.md
    │   │       │   ├── sede_ineachdb.md
    │   │       │   ├── sede_sites.md
    │   │       │   ├── sede_tables.md
    │   │       │   ├── sede_users.md
    │   │       │   ├── suggested_edit_votes.md
    │   │       │   ├── suggested_edits.md
    │   │       │   ├── tag_synonyms.md
    │   │       │   └── vote_types.md
    │   │       ├── tables
    │   │       │   ├── badges.md
    │   │       │   ├── comments.md
    │   │       │   ├── index.md
    │   │       │   ├── post_history.md
    │   │       │   ├── post_links.md
    │   │       │   ├── posts_answers.md
    │   │       │   ├── posts_moderator_nomination.md
    │   │       │   ├── posts_orphaned_tag_wiki.md
    │   │       │   ├── posts_privilege_wiki.md
    │   │       │   ├── posts_questions.md
    │   │       │   ├── posts_tag_wiki.md
    │   │       │   ├── posts_tag_wiki_excerpt.md
    │   │       │   ├── posts_wiki_placeholder.md
    │   │       │   ├── stackoverflow_posts.md
    │   │       │   ├── tags.md
    │   │       │   ├── users.md
    │   │       │   └── votes.md
    │   │       ├── index.md
    │   │       └── viz.html
    │   ├── samples
    │   │   ├── crypto_bitcoin
    │   │   │   ├── README.md
    │   │   │   └── seeds.txt
    │   │   ├── ga4_merch_store
    │   │   │   ├── README.md
    │   │   │   └── seeds.txt
    │   │   └── stackoverflow
    │   │       ├── README.md
    │   │       └── seeds.txt
    │   ├── src
    │   │   └── reference_agent
    │   │       ├── bundle
    │   │       │   ├── __init__.py
    │   │       │   ├── document.py
    │   │       │   ├── index.py
    │   │       │   ├── paths.py
    │   │       │   └── synthesizer.py
    │   │       ├── prompts
    │   │       │   ├── reference_instruction.md
    │   │       │   └── web_ingestion_instruction.md
    │   │       ├── sources
    │   │       │   ├── __init__.py
    │   │       │   ├── base.py
    │   │       │   └── bigquery.py
    │   │       ├── tools
    │   │       │   ├── __init__.py
    │   │       │   ├── bundle_tools.py
    │   │       │   ├── context.py
    │   │       │   ├── source_tools.py
    │   │       │   └── web_tools.py
    │   │       ├── viewer
    │   │       │   ├── static
    │   │       │   │   ├── viz.css
    │   │       │   │   └── viz.js
    │   │       │   ├── templates
    │   │       │   │   └── viz.html
    │   │       │   ├── __init__.py
    │   │       │   └── generator.py
    │   │       ├── web
    │   │       │   ├── __init__.py
    │   │       │   └── fetcher.py
    │   │       ├── __init__.py
    │   │       ├── __main__.py
    │   │       ├── agent.py
    │   │       ├── cli.py
    │   │       └── runner.py
    │   ├── tests
    │   │   ├── __init__.py
    │   │   ├── test_bigquery_source.py
    │   │   ├── test_bundle_tools.py
    │   │   ├── test_document.py
    │   │   ├── test_index.py
    │   │   ├── test_viewer.py
    │   │   ├── test_web_fetcher.py
    │   │   └── test_web_tools.py
    │   ├── .gitignore
    │   ├── LICENSE.md
    │   ├── pyproject.toml
    │   ├── README.md
    │   └── SPEC.md
    ├── samples
    │   ├── discovery
    │   │   ├── agent.py
    │   │   ├── README.md
    │   │   ├── requirements.txt
    │   │   ├── SKILL.md
    │   │   ├── tools.py
    │   │   └── utils.py
    │   ├── enrichment
    │   │   ├── sample
    │   │   │   ├── config
    │   │   │   │   ├── skills
    │   │   │   │   │   └── kb-search
    │   │   │   │   │       └── SKILL.md
    │   │   │   │   └── instructions.md
    │   │   │   ├── data
    │   │   │   │   └── create_data.py
    │   │   │   └── docs
    │   │   │       ├── events.md
    │   │   │       ├── example1.md
    │   │   │       ├── example2.md
    │   │   │       ├── example3.md
    │   │   │       ├── example4.md
    │   │   │       ├── lipsum.md
    │   │   │       ├── overview.md
    │   │   │       └── usage.md
    │   │   ├── src
    │   │   │   ├── enrichment
    │   │   │   │   ├── documentation
    │   │   │   │   │   ├── __init__.py
    │   │   │   │   │   ├── agent.md
    │   │   │   │   │   ├── agent.py
    │   │   │   │   │   └── sources.py
    │   │   │   │   ├── metadata
    │   │   │   │   │   ├── __init__.py
    │   │   │   │   │   ├── catalog.py
    │   │   │   │   │   └── snapshot.py
    │   │   │   │   ├── util
    │   │   │   │   │   ├── __init__.py
    │   │   │   │   │   └── markdown.py
    │   │   │   │   ├── __init__.py
    │   │   │   │   ├── download.py
    │   │   │   │   ├── enrich.py
    │   │   │   │   └── publish.py
    │   │   │   ├── tools
    │   │   │   │   └── fileskb
    │   │   │   │       ├── main.py
    │   │   │   │       └── README.md
    │   │   │   ├── env.sh
    │   │   │   └── requirements.txt
    │   │   ├── .gitignore
    │   │   └── README.md
    │   └── README.md
    ├── toolbox
    │   ├── enrichment
    │   │   ├── src
    │   │   │   ├── agent
    │   │   │   │   ├── enrich
    │   │   │   │   │   ├── agent.ts
    │   │   │   │   │   └── command.ts
    │   │   │   │   ├── utils
    │   │   │   │   │   ├── patchadk.ts
    │   │   │   │   │   └── patchpb.ts
    │   │   │   │   ├── main.ts
    │   │   │   │   └── tools.ts
    │   │   │   └── tools
    │   │   │       └── md
    │   │   │           ├── fileset.ts
    │   │   │           ├── main.ts
    │   │   │           └── server.ts
    │   │   ├── .gitignore
    │   │   └── README.md
    │   ├── mdcode
    │   │   ├── demo
    │   │   │   ├── bq
    │   │   │   │   ├── cleanup.ts
    │   │   │   │   ├── setup.ts
    │   │   │   │   └── update.ts
    │   │   │   ├── kb
    │   │   │   │   ├── cleanup.ts
    │   │   │   │   ├── setup.ts
    │   │   │   │   └── update.ts
    │   │   │   ├── okf
    │   │   │   │   ├── catalog
    │   │   │   │   │   ├── datasets
    │   │   │   │   │   │   ├── ga4_obfuscated_sample_ecommerce.md
    │   │   │   │   │   │   └── index.md
    │   │   │   │   │   ├── references
    │   │   │   │   │   │   ├── joins
    │   │   │   │   │   │   │   ├── events___ads_clickstats.md
    │   │   │   │   │   │   │   └── index.md
    │   │   │   │   │   │   ├── metrics
    │   │   │   │   │   │   │   ├── avg_pageviews.md
    │   │   │   │   │   │   │   ├── avg_spend_per_purchase_session_by_user.md
    │   │   │   │   │   │   │   ├── avg_transactions_per_purchaser.md
    │   │   │   │   │   │   │   ├── day_count.md
    │   │   │   │   │   │   │   ├── event_count.md
    │   │   │   │   │   │   │   ├── index.md
    │   │   │   │   │   │   │   ├── new_user_count.md
    │   │   │   │   │   │   │   ├── overall_avg_spend_per_purchase_session.md
    │   │   │   │   │   │   │   └── user_count.md
    │   │   │   │   │   │   └── index.md
    │   │   │   │   │   ├── tables
    │   │   │   │   │   │   ├── events_.md
    │   │   │   │   │   │   └── index.md
    │   │   │   │   │   └── index.md
    │   │   │   │   ├── cleanup.ts
    │   │   │   │   └── setup.ts
    │   │   │   ├── .gitignore
    │   │   │   └── README.md
    │   │   ├── docs
    │   │   │   ├── concept.md
    │   │   │   ├── design.md
    │   │   │   ├── plan.md
    │   │   │   └── spec.md
    │   │   ├── src
    │   │   │   ├── libts
    │   │   │   │   ├── gcp
    │   │   │   │   │   ├── api.ts
    │   │   │   │   │   ├── bigquery.ts
    │   │   │   │   │   ├── context.ts
    │   │   │   │   │   ├── crm.ts
    │   │   │   │   │   ├── dataplex.ts
    │   │   │   │   │   └── index.ts
    │   │   │   │   ├── layouts
    │   │   │   │   │   ├── documents.ts
    │   │   │   │   │   └── standard.ts
    │   │   │   │   ├── sources
    │   │   │   │   │   ├── bq-dataset.ts
    │   │   │   │   │   ├── entrygroup.ts
    │   │   │   │   │   └── kb.ts
    │   │   │   │   ├── index.ts
    │   │   │   │   ├── layout.ts
    │   │   │   │   ├── manifest.ts
    │   │   │   │   ├── metadata.ts
    │   │   │   │   ├── snapshot.ts
    │   │   │   │   ├── source.ts
    │   │   │   │   └── sync.ts
    │   │   │   └── tool
    │   │   │       ├── commands.ts
    │   │   │       ├── main.ts
    │   │   │       └── mcp.ts
    │   │   ├── tests
    │   │   │   ├── libts
    │   │   │   │   ├── mocks.ts
    │   │   │   │   └── scenarios.ts
    │   │   │   └── scenarios
    │   │   ├── .gitignore
    │   │   └── README.md
    │   └── README.md
    ├── .gitignore
    ├── CODE_OF_CONDUCT.md
    ├── CONTRIBUTING.md
    ├── LICENSE.md
    └── README.md


================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Code of Conduct

## Our Pledge

In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of
experience, education, socio-economic status, nationality, personal appearance,
race, religion, or sexual identity and orientation.

## Our Standards

Examples of behavior that contributes to creating a positive environment
include:

* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members

Examples of unacceptable behavior by participants include:

* The use of sexualized language or imagery and unwelcome sexual attention or
    advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
    address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
    professional setting

## Our Responsibilities

Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.

Project maintainers have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, or to ban temporarily or permanently any
contributor for other behaviors that they deem inappropriate, threatening,
offensive, or harmful.

## Scope

This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.

This Code of Conduct also applies outside the project spaces when the Project
Steward has a reasonable belief that an individual's behavior may have a
negative impact on the project or its community.

## Conflict Resolution

We do not believe that all conflict is bad; healthy debate and disagreement
often yield positive results. However, it is never okay to be disrespectful or
to engage in behavior that violates the project’s code of conduct.

If you see someone violating the code of conduct, you are encouraged to address
the behavior directly with those involved. Many issues can be resolved quickly
and easily, and this gives people more control over the outcome of their
dispute. If you are unable to resolve the matter for any reason, or if the
behavior is threatening or harassing, report it. We are dedicated to providing
an environment where participants feel welcome and safe.

Reports should be directed to @googleapis/senseai-eco, the
Project Steward(s) for *Project Toolbox*. It is the Project Steward’s duty to
receive and address reported violations of the code of conduct. They will then
work with a committee consisting of representatives from the Open Source
Programs Office and the Google Open Source Strategy team. If for any reason you
are uncomfortable reaching out to the Project Steward, please email
<opensource@google.com>.

We will investigate every complaint, but you may not receive a direct response.
We will use our discretion in determining when and how to follow up on reported
incidents, which may range from not taking action to permanent expulsion from
the project and project-sponsored spaces. We will notify the accused of the
report and provide them an opportunity to discuss it before any action is taken.
The identity of the reporter will be omitted from the details of the report
supplied to the accused. In potentially harmful situations, such as ongoing
harassment or threats to anyone's safety, we may take action without notice.

## Attribution

This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
available at
<https://www.contributor-covenant.org/version/1/4/code-of-conduct.html>


================================================
FILE: CONTRIBUTING.md
================================================
# Contributing

Thanks for your interest in contributing to the Google Cloud Knowledge Catalog
repo!

To get started contributing:

1. Sign a Contributor License Agreement (see details below).
1. Fork the repo, develop and test your code changes.
1. Ensure that your code adheres to the existing style.
1. Ensure that your code has an appropriate set of unit tests which all pass.
1. Ensure that all tests pass (e.g., by running `npm run test` in the TypeScript package directories).
1. Submit a pull request.

## Contributor License Agreement

Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution;
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.

You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.

## Code reviews

All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
information on using pull requests.



================================================
FILE: LICENSE.md
================================================

                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/

   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

   1. Definitions.

      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.

      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.

      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.

      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.

      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.

      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).

      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.

      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."

      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.

   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.

   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:

      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and

      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and

      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and

      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.

      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.

   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.

   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.

   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.

   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.

   END OF TERMS AND CONDITIONS

   APPENDIX: How to apply the Apache License to your work.

      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

   Copyright [yyyy] [name of copyright owner]

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.



================================================
FILE: okf\bundles\crypto_bitcoin\datasets\crypto_bitcoin.md
================================================
---
type: BigQuery Dataset
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin
title: Cryptocurrency Bitcoin
description: This BigQuery public dataset contains a complete history of the Bitcoin
  blockchain and updates every 10 minutes.
tags:
- cryptocurrency
- bitcoin
- blockchain
- public data
- gcp
- data-analytics
timestamp: '2026-05-28T22:44:47+00:00'
---

The `crypto_bitcoin` dataset provides a comprehensive and up-to-date record of the entire Bitcoin blockchain. It includes detailed information about [blocks](../tables/blocks.md), [transactions](../tables/transactions.md), transaction [inputs](../tables/inputs.md), and [outputs](../tables/outputs.md). This dataset is part of the BigQuery Public Datasets program, making it freely accessible for analysis and research into Bitcoin's operations, economics, and historical trends. Researchers, developers, and enthusiasts can use this data to understand transaction patterns, network activity, and the overall state of the Bitcoin network.

# Schema

This dataset contains the following tables, providing a complete history of the Bitcoin blockchain:

*   [blocks](../tables/blocks.md)
*   [inputs](../tables/inputs.md)
*   [outputs](../tables/outputs.md)
*   [transactions](../tables/transactions.md)

# Common query patterns

```sql
-- Count the total number of blocks in the Bitcoin blockchain
SELECT
    COUNT(*)
FROM
    `bigquery-public-data.crypto_bitcoin.blocks`;
```

```sql
-- Get the total number of transactions over time
SELECT
    DATE(block_timestamp) AS transaction_date,
    COUNT(transaction_id) AS total_transactions
FROM
    `bigquery-public-data.crypto_bitcoin.transactions`
GROUP BY
    transaction_date
ORDER BY
    transaction_date DESC
LIMIT 100;
```

# Citations

[1] [BigQuery Public Dataset: crypto_bitcoin](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin)
[2] [Bitcoin in BigQuery: blockchain analytics on public data](https://cloud.google.com/blog/products/gcp/bitcoin-in-bigquery-blockchain-analytics-on-public-data)


================================================
FILE: okf\bundles\crypto_bitcoin\datasets\index.md
================================================
# BigQuery Dataset

* [Cryptocurrency Bitcoin](crypto_bitcoin.md) - This BigQuery public dataset contains a complete history of the Bitcoin blockchain and updates every 10 minutes.


================================================
FILE: okf\bundles\crypto_bitcoin\index.md
================================================
# Subdirectories

* [datasets](datasets/index.md) - This BigQuery public dataset contains a complete history of the Bitcoin blockchain and updates every 10 minutes.
* [tables](tables/index.md) - This directory contains BigQuery table details for Bitcoin blocks, transactions, inputs, and outputs.


================================================
FILE: okf\bundles\crypto_bitcoin\tables\blocks.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/blocks
title: Bitcoin Blocks Table
description: Details about the Bitcoin Blocks BigQuery table, including its schema.
tags:
- bitcoin
- bigquery
- blocks
- blockchain
timestamp: '2026-05-28T22:43:59+00:00'
---

# Schema

| Field | Type |
| --- | --- |
| hash | hex_string |
| size | bigint |
| stripped_size | bigint |
| weight | bigint |
| number | bigint |
| version | bigint |
| merkle_root | hex_string |
| timestamp | bigint |
| nonce | hex_string |
| bits | hex_string |
| coinbase_param | hex_string |
| transaction_count | bigint |

# Citations
- https://github.com/blockchain-etl/bitcoin-etl


================================================
FILE: okf\bundles\crypto_bitcoin\tables\index.md
================================================
# BigQuery Table

* [Bitcoin Blocks Table](blocks.md) - Details about the Bitcoin Blocks BigQuery table, including its schema.
* [Bitcoin Transaction Inputs](inputs.md) - Details about transaction inputs on the Bitcoin blockchain.
* [Bitcoin Transactions](transactions.md) - A comprehensive table detailing all transactions on the Bitcoin blockchain.
* [Outputs](outputs.md) - Outputs from all transactions in the Bitcoin blockchain.


================================================
FILE: okf\bundles\crypto_bitcoin\tables\inputs.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/inputs
title: Bitcoin Transaction Inputs
description: Details about transaction inputs on the Bitcoin blockchain.
tags:
- bitcoin
- blockchain
- cryptocurrency
- transactions
- inputs
- etl
timestamp: '2026-05-28T22:44:24+00:00'
---

This table, part of the public [crypto_bitcoin](../datasets/crypto_bitcoin.md) dataset, contains detailed information about every input used in Bitcoin transactions. Each row represents a single transaction input, which typically references an unspent output from a previous transaction. This table is crucial for tracing the flow of Bitcoin and understanding the history of transactions. It records where the coins originated (`spent_transaction_hash` and `spent_output_index`) and the associated `value` transferred. This table can be joined with the [transactions](transactions.md) table on `transaction_hash` and [outputs](outputs.md) to reconstruct the full transaction graph.

# Schema

*   `transaction_hash`: STRING
*   `block_hash`: STRING
*   `block_number`: INTEGER
*   `block_timestamp`: TIMESTAMP
*   `index`: INTEGER
*   `spent_transaction_hash`: STRING
*   `spent_output_index`: INTEGER
*   `script_asm`: STRING
*   `script_hex`: STRING
*   `sequence`: INTEGER
*   `required_signatures`: INTEGER
*   `type`: STRING
*   `addresses`: REPEATED STRING
*   `value`: NUMERIC

# Common query patterns

```sql
SELECT *
FROM `bigquery-public-data.crypto_bitcoin.inputs`
WHERE transaction_hash = 'YOUR_TRANSACTION_HASH_HERE'
LIMIT 10
```

```sql
SELECT
    block_number,
    SUM(value) AS total_input_value
FROM `bigquery-public-data.crypto_bitcoin.inputs`
WHERE block_number = 600000 -- Example block number
GROUP BY block_number
```

```sql
SELECT DISTINCT
    address
FROM `bigquery-public-data.crypto_bitcoin.inputs`,
    UNNEST(addresses) AS address
WHERE block_timestamp >= '2023-01-01'
LIMIT 10
```

# Citations

[1] [BigQuery Table: inputs](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/inputs)
[2] [blockchain-etl/bitcoin-etl](https://github.com/blockchain-etl/bitcoin-etl)


================================================
FILE: okf\bundles\crypto_bitcoin\tables\outputs.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs
title: Outputs
description: Outputs from all transactions in the Bitcoin blockchain.
tags:
- bitcoin
- blockchain
- transactions
- outputs
- etl
timestamp: '2026-05-28T22:44:32+00:00'
---

The `outputs` table contains records of all transaction outputs within the Bitcoin blockchain. Each row in this table represents a single output from a Bitcoin transaction, detailing the amount transferred, the destination addresses, and other script-related information. This table is crucial for understanding the flow of Bitcoin and analyzing transaction patterns, especially when linked with the `[transactions](transactions.md)` and `[inputs](inputs.md)` tables.

# Schema

- `transaction_hash`: The hash of the transaction this output belongs to.
- `block_hash`: The hash of the block containing this transaction.
- `block_number`: The number of the block containing this transaction.
- `block_timestamp`: The timestamp of the block containing this transaction.
- `index`: The zero-based index of this output within its transaction.
- `script_asm`: The script in assembly format.
- `script_hex`: The script in hexadecimal format.
- `required_signatures`: The number of signatures required to spend this output.
- `type`: The type of the output script.
- `addresses`: (REPEATED) Array of destination addresses for this output.
- `value`: The value of the output in satoshis.

# Common query patterns

```sql
SELECT
  t.*
FROM
  `bigquery-public-data.crypto_bitcoin.outputs` AS t
WHERE
  t.transaction_hash = 'some_transaction_hash'
```

```sql
SELECT
  SUM(t.value) AS total_output_value
FROM
  `bigquery-public-data.crypto_bitcoin.outputs` AS t
WHERE
  t.block_number = 123456
```

```sql
SELECT
  t.type,
  COUNT(*) AS output_count
FROM
  `bigquery-public-data.crypto_bitcoin.outputs` AS t
WHERE
  t.block_timestamp BETWEEN TIMESTAMP('2023-01-01') AND TIMESTAMP('2023-01-31')
GROUP BY
  t.type
ORDER BY
  output_count DESC
```

# Citations

[1] [Outputs Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs)
[2] [Bitcoin ETL on GitHub](https://github.com/blockchain-etl/bitcoin-etl)


================================================
FILE: okf\bundles\crypto_bitcoin\tables\transactions.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions
title: Bitcoin Transactions
description: A comprehensive table detailing all transactions on the Bitcoin blockchain.
tags:
- bitcoin
- blockchain
- transactions
- crypto
- public data
- etl
timestamp: '2026-05-28T22:45:04+00:00'
---

The `transactions` table in the [crypto_bitcoin](../datasets/crypto_bitcoin.md) dataset provides a complete record of every transaction ever processed on the Bitcoin blockchain. Each row represents a single transaction, offering granular details such as its hash, size, associated [block](blocks.md) information (hash, number, timestamp), and the total input and output values. Importantly, it includes detailed arrays for both [inputs](inputs.md) and [outputs](outputs.md), each specifying spent transaction details, script information, involved addresses, and values. This table is essential for in-depth analysis of transaction flows, tracing funds, and understanding the economic activity within the Bitcoin network. The grain is one row per transaction, with data spanning the entire history of the Bitcoin blockchain, partitioned by `block_timestamp_month`.

# Schema
- `hash` STRING REQUIRED: The hash of this transaction
- `size` INTEGER: The size of this transaction in bytes
- `virtual_size` INTEGER: The virtual transaction size (differs from size for witness transactions)
- `version` INTEGER: Protocol version specified in block which contained this transaction
- `lock_time` INTEGER: Earliest time that miners can include the transaction in their hashing of the Merkle root to attach it in the latest block of the blockchain
- `block_hash` STRING REQUIRED: Hash of the block which contains this transaction
- `block_number` INTEGER REQUIRED: Number of the block which contains this transaction
- `block_timestamp` TIMESTAMP REQUIRED: Timestamp of the block which contains this transaction
- `block_timestamp_month` DATE REQUIRED: Month of the block which contains this transaction
- `input_count` INTEGER: The number of inputs in the transaction
- `output_count` INTEGER: The number of outputs in the transaction
- `input_value` NUMERIC: Total value of inputs in the transaction
- `output_value` NUMERIC: Total value of outputs in the transaction
- `is_coinbase` BOOLEAN: True if this transaction is a coinbase transaction
- `fee` NUMERIC: The fee paid by this transaction
- `inputs` RECORD REPEATED: Transaction inputs
  - `index` INTEGER REQUIRED: 0-indexed number of an input within a transaction
  - `spent_transaction_hash` STRING: The hash of the transaction which contains the output that this input spends
  - `spent_output_index` INTEGER: The index of the output this input spends
  - `script_asm` STRING: Symbolic representation of the bitcoin's script language op-codes
  - `script_hex` STRING: Hexadecimal representation of the bitcoin's script language op-codes
  - `sequence` INTEGER: A number intended to allow unconfirmed time-locked transactions to be updated before being finalized
  - `required_signatures` INTEGER: The number of signatures required to authorize the spent output
  - `type` STRING: The address type of the spent output
  - `addresses` STRING REPEATED: Addresses which own the spent output
  - `value` NUMERIC: The value in base currency attached to the spent output
- `outputs` RECORD REPEATED: Transaction outputs
  - `index` INTEGER REQUIRED: 0-indexed number of an output within a transaction
  - `script_asm` STRING: Symbolic representation of the bitcoin's script language op-codes
  - `script_hex` STRING: Hexadecimal representation of the bitcoin's script language op-codes
  - `required_signatures` INTEGER: The number of signatures required to authorize spending of this output
  - `type` STRING: The address type of the output
  - `addresses` STRING REPEATED: Addresses which own this output
  - `value` NUMERIC: The value in base currency attached to this output

# Common query patterns
```sql
-- Get the total number of transactions per day
SELECT
    DATE(block_timestamp) AS transaction_date,
    COUNT(hash) AS transaction_count
FROM
    `bigquery-public-data.crypto_bitcoin.transactions`
WHERE
    block_timestamp BETWEEN '2023-01-01' AND '2023-01-31'
GROUP BY
    transaction_date
ORDER BY
    transaction_date DESC;
```
```sql
-- Find transactions involving a specific address as an output
SELECT
    t.hash AS transaction_hash,
    t.block_timestamp
FROM
    `bigquery-public-data.crypto_bitcoin.transactions` AS t,
    UNNEST(t.outputs) AS output
WHERE
    '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa' IN UNNEST(output.addresses)
LIMIT 10;
```
```sql
-- Calculate the total fees collected in a given month
SELECT
    FORMAT_TIMESTAMP('%Y-%m', block_timestamp) AS transaction_month,
    SUM(fee) AS total_fee
FROM
    `bigquery-public-data.crypto_bitcoin.transactions`
WHERE
    block_timestamp BETWEEN '2023-01-01' AND '2023-01-31'
GROUP BY
    transaction_month;
```
```sql
-- Find duplicate transactions (anomaly detection)
SELECT
   *
FROM (
 SELECT
   hash,
   COUNT(hash) AS dup_transaction_count
 FROM
   `bigquery-public-data.crypto_bitcoin.transactions`
 GROUP BY
   hash)
WHERE
 dup_transaction_count > 1
```

# Citations
[1] [Bitcoin Transactions](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions)
[2] [Bitcoin ETL](https://github.com/blockchain-etl/bitcoin-etl)
[3] [Bitcoin in BigQuery: blockchain analytics on public data](https://cloud.google.com/blog/products/gcp/bitcoin-in-bigquery-blockchain-analytics-on-public-data)


================================================
FILE: okf\bundles\crypto_bitcoin\viz.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>OKF Bundle Viewer</title>
<script src="https://cdn.jsdelivr.net/npm/cytoscape@3.28.1/dist/cytoscape.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script>
<style>
* { box-sizing: border-box; }
body {
  margin: 0;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
  font-size: 14px;
  color: #0f172a;
  background: #f8fafc;
  display: flex;
  flex-direction: column;
  height: 100vh;
}
header {
  display: flex;
  align-items: center;
  justify-content: space-between;
  padding: 10px 16px;
  background: #fff;
  border-bottom: 1px solid #e2e8f0;
  flex-shrink: 0;
}
.title strong { font-size: 16px; margin-right: 8px; }
.muted { color: #64748b; font-size: 12px; }
.controls { display: flex; gap: 8px; }
.controls input, .controls select, .controls button {
  font-size: 13px;
  padding: 5px 8px;
  border: 1px solid #cbd5e1;
  border-radius: 4px;
  background: #fff;
}
.controls input { width: 220px; }
.controls button { cursor: pointer; background: #f1f5f9; }
.controls button:hover { background: #e2e8f0; }

main {
  display: flex;
  flex: 1;
  min-height: 0;
}
#graph {
  flex: 1 1 60%;
  background: #fff;
  border-right: 1px solid #e2e8f0;
  min-width: 0;
  position: relative;
}
#detail {
  flex: 0 0 40%;
  overflow-y: auto;
  padding: 18px 22px;
  background: #fff;
}
#detail-empty {
  text-align: center;
  margin-top: 40px;
}

.detail-header { margin-bottom: 12px; }
.detail-header h1 {
  font-size: 18px;
  margin: 4px 0 2px;
  font-weight: 600;
}
.type-chip {
  display: inline-block;
  padding: 2px 8px;
  border-radius: 10px;
  font-size: 11px;
  font-weight: 600;
  color: #fff;
  background: #94a3b8;
  text-transform: uppercase;
  letter-spacing: 0.5px;
}
dl.frontmatter {
  display: grid;
  grid-template-columns: 90px 1fr;
  row-gap: 4px;
  column-gap: 12px;
  margin: 8px 0 12px;
  font-size: 13px;
}
dl.frontmatter dt {
  color: #64748b;
  font-weight: 500;
}
dl.frontmatter dd { margin: 0; }
dl.frontmatter a { color: #2563eb; word-break: break-all; }

.tag {
  display: inline-block;
  padding: 1px 6px;
  margin: 0 4px 2px 0;
  border-radius: 4px;
  background: #f1f5f9;
  color: #475569;
  font-size: 11px;
}

hr { border: none; border-top: 1px solid #e2e8f0; margin: 14px 0; }

#detail-body { font-size: 13px; line-height: 1.55; }
#detail-body h1 {
  font-size: 16px; margin: 18px 0 6px;
  padding-bottom: 4px; border-bottom: 1px solid #e2e8f0;
}
#detail-body h2 { font-size: 14px; margin: 14px 0 4px; }
#detail-body h3 { font-size: 13px; margin: 12px 0 4px; }
#detail-body p { margin: 6px 0; }
#detail-body code {
  background: #f1f5f9;
  padding: 1px 4px;
  border-radius: 3px;
  font-size: 12px;
  font-family: ui-monospace, "SF Mono", Consolas, monospace;
}
#detail-body pre {
  background: #0f172a;
  color: #e2e8f0;
  padding: 10px 12px;
  border-radius: 6px;
  overflow-x: auto;
  font-size: 12px;
}
#detail-body pre code { background: transparent; color: inherit; padding: 0; }
#detail-body ul, #detail-body ol { padding-left: 22px; margin: 6px 0; }
#detail-body li { margin: 2px 0; }
#detail-body table { border-collapse: collapse; margin: 8px 0; }
#detail-body th, #detail-body td {
  border: 1px solid #e2e8f0; padding: 4px 8px; font-size: 12px;
}
#detail-body a.internal { color: #2563eb; cursor: pointer; }
#detail-body a.external { color: #2563eb; }

#detail-backlinks { margin-top: 18px; }
#detail-backlinks h2 { font-size: 13px; color: #64748b; margin-bottom: 6px; }
#detail-backlinks ul { padding-left: 18px; }
#detail-backlinks a { color: #2563eb; cursor: pointer; }

</style>
</head>
<body>
<header>
  <div class="title">
    <strong id="bundle-name"></strong>
    <span class="muted">OKF bundle</span>
  </div>
  <div class="controls">
    <input id="search" type="search" placeholder="Search title / id / tag">
    <select id="filter-type">
      <option value="">All types</option>
    </select>
    <select id="layout">
      <option value="cose">cose (force)</option>
      <option value="concentric">concentric</option>
      <option value="breadthfirst">breadth-first</option>
      <option value="circle">circle</option>
      <option value="grid">grid</option>
    </select>
    <button id="reset">Reset view</button>
  </div>
</header>

<main>
  <section id="graph"></section>
  <section id="detail">
    <div id="detail-empty" class="muted">Click a node to see its details.</div>
    <article id="detail-content" hidden>
      <header class="detail-header">
        <span class="type-chip" id="detail-type"></span>
        <h1 id="detail-title"></h1>
        <div class="muted" id="detail-id"></div>
      </header>
      <dl class="frontmatter">
        <dt>Description</dt><dd id="detail-description"></dd>
        <dt>Resource</dt><dd id="detail-resource"></dd>
        <dt>Tags</dt><dd id="detail-tags"></dd>
      </dl>
      <hr>
      <div id="detail-body"></div>
      <section id="detail-backlinks" hidden>
        <h2>Cited by</h2>
        <ul id="backlinks-list"></ul>
      </section>
    </article>
  </section>
</main>

<script>
window.BUNDLE_NAME = "crypto_bitcoin";
window.BUNDLE = {"nodes": [{"data": {"id": "datasets/crypto_bitcoin", "label": "Cryptocurrency Bitcoin", "type": "BigQuery Dataset", "description": "This BigQuery public dataset contains a complete history of the Bitcoin blockchain and updates every 10 minutes.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin", "tags": ["cryptocurrency", "bitcoin", "blockchain", "public data", "gcp", "data-analytics"], "color": "#8b5cf6", "size": 38}}, {"data": {"id": "tables/blocks", "label": "Bitcoin Blocks Table", "type": "BigQuery Table", "description": "Details about the Bitcoin Blocks BigQuery table, including its schema.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/blocks", "tags": ["bitcoin", "bigquery", "blocks", "blockchain"], "color": "#3b82f6", "size": 31}}, {"data": {"id": "tables/inputs", "label": "Bitcoin Transaction Inputs", "type": "BigQuery Table", "description": "Details about transaction inputs on the Bitcoin blockchain.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/inputs", "tags": ["bitcoin", "blockchain", "cryptocurrency", "transactions", "inputs", "etl"], "color": "#3b82f6", "size": 39}}, {"data": {"id": "tables/outputs", "label": "Outputs", "type": "BigQuery Table", "description": "Outputs from all transactions in the Bitcoin blockchain.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs", "tags": ["bitcoin", "blockchain", "transactions", "outputs", "etl"], "color": "#3b82f6", "size": 39}}, {"data": {"id": "tables/transactions", "label": "Bitcoin Transactions", "type": "BigQuery Table", "description": "A comprehensive table detailing all transactions on the Bitcoin blockchain.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions", "tags": ["bitcoin", "blockchain", "transactions", "crypto", "public data", "etl"], "color": "#3b82f6", "size": 56}}], "edges": [{"data": {"id": "datasets/crypto_bitcoin__tables/blocks", "source": "datasets/crypto_bitcoin", "target": "tables/blocks"}}, {"data": {"id": "datasets/crypto_bitcoin__tables/transactions", "source": "datasets/crypto_bitcoin", "target": "tables/transactions"}}, {"data": {"id": "datasets/crypto_bitcoin__tables/inputs", "source": "datasets/crypto_bitcoin", "target": "tables/inputs"}}, {"data": {"id": "datasets/crypto_bitcoin__tables/outputs", "source": "datasets/crypto_bitcoin", "target": "tables/outputs"}}, {"data": {"id": "tables/inputs__datasets/crypto_bitcoin", "source": "tables/inputs", "target": "datasets/crypto_bitcoin"}}, {"data": {"id": "tables/inputs__tables/transactions", "source": "tables/inputs", "target": "tables/transactions"}}, {"data": {"id": "tables/inputs__tables/outputs", "source": "tables/inputs", "target": "tables/outputs"}}, {"data": {"id": "tables/outputs__tables/transactions", "source": "tables/outputs", "target": "tables/transactions"}}, {"data": {"id": "tables/outputs__tables/inputs", "source": "tables/outputs", "target": "tables/inputs"}}, {"data": {"id": "tables/transactions__datasets/crypto_bitcoin", "source": "tables/transactions", "target": "datasets/crypto_bitcoin"}}, {"data": {"id": "tables/transactions__tables/blocks", "source": "tables/transactions", "target": "tables/blocks"}}, {"data": {"id": "tables/transactions__tables/inputs", "source": "tables/transactions", "target": "tables/inputs"}}, {"data": {"id": "tables/transactions__tables/outputs", "source": "tables/transactions", "target": "tables/outputs"}}], "bodies": {"datasets/crypto_bitcoin": "The `crypto_bitcoin` dataset provides a comprehensive and up-to-date record of the entire Bitcoin blockchain. It includes detailed information about [blocks](/tables/blocks.md), [transactions](/tables/transactions.md), transaction [inputs](/tables/inputs.md), and [outputs](/tables/outputs.md). This dataset is part of the BigQuery Public Datasets program, making it freely accessible for analysis and research into Bitcoin's operations, economics, and historical trends. Researchers, developers, and enthusiasts can use this data to understand transaction patterns, network activity, and the overall state of the Bitcoin network.\n\n# Schema\n\nThis dataset contains the following tables, providing a complete history of the Bitcoin blockchain:\n\n*   [blocks](/tables/blocks.md)\n*   [inputs](/tables/inputs.md)\n*   [outputs](/tables/outputs.md)\n*   [transactions](/tables/transactions.md)\n\n# Common query patterns\n\n```sql\n-- Count the total number of blocks in the Bitcoin blockchain\nSELECT\n    COUNT(*)\nFROM\n    `bigquery-public-data.crypto_bitcoin.blocks`;\n```\n\n```sql\n-- Get the total number of transactions over time\nSELECT\n    DATE(block_timestamp) AS transaction_date,\n    COUNT(transaction_id) AS total_transactions\nFROM\n    `bigquery-public-data.crypto_bitcoin.transactions`\nGROUP BY\n    transaction_date\nORDER BY\n    transaction_date DESC\nLIMIT 100;\n```\n\n# Citations\n\n[1] [BigQuery Public Dataset: crypto_bitcoin](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin)\n[2] [Bitcoin in BigQuery: blockchain analytics on public data](https://cloud.google.com/blog/products/gcp/bitcoin-in-bigquery-blockchain-analytics-on-public-data)", "tables/blocks": "# Schema\n\n| Field | Type |\n| --- | --- |\n| hash | hex_string |\n| size | bigint |\n| stripped_size | bigint |\n| weight | bigint |\n| number | bigint |\n| version | bigint |\n| merkle_root | hex_string |\n| timestamp | bigint |\n| nonce | hex_string |\n| bits | hex_string |\n| coinbase_param | hex_string |\n| transaction_count | bigint |\n\n# Citations\n- https://github.com/blockchain-etl/bitcoin-etl", "tables/inputs": "This table, part of the public [crypto_bitcoin](/datasets/crypto_bitcoin.md) dataset, contains detailed information about every input used in Bitcoin transactions. Each row represents a single transaction input, which typically references an unspent output from a previous transaction. This table is crucial for tracing the flow of Bitcoin and understanding the history of transactions. It records where the coins originated (`spent_transaction_hash` and `spent_output_index`) and the associated `value` transferred. This table can be joined with the [transactions](/tables/transactions.md) table on `transaction_hash` and [outputs](/tables/outputs.md) to reconstruct the full transaction graph.\n\n# Schema\n\n*   `transaction_hash`: STRING\n*   `block_hash`: STRING\n*   `block_number`: INTEGER\n*   `block_timestamp`: TIMESTAMP\n*   `index`: INTEGER\n*   `spent_transaction_hash`: STRING\n*   `spent_output_index`: INTEGER\n*   `script_asm`: STRING\n*   `script_hex`: STRING\n*   `sequence`: INTEGER\n*   `required_signatures`: INTEGER\n*   `type`: STRING\n*   `addresses`: REPEATED STRING\n*   `value`: NUMERIC\n\n# Common query patterns\n\n```sql\nSELECT *\nFROM `bigquery-public-data.crypto_bitcoin.inputs`\nWHERE transaction_hash = 'YOUR_TRANSACTION_HASH_HERE'\nLIMIT 10\n```\n\n```sql\nSELECT\n    block_number,\n    SUM(value) AS total_input_value\nFROM `bigquery-public-data.crypto_bitcoin.inputs`\nWHERE block_number = 600000 -- Example block number\nGROUP BY block_number\n```\n\n```sql\nSELECT DISTINCT\n    address\nFROM `bigquery-public-data.crypto_bitcoin.inputs`,\n    UNNEST(addresses) AS address\nWHERE block_timestamp >= '2023-01-01'\nLIMIT 10\n```\n\n# Citations\n\n[1] [BigQuery Table: inputs](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/inputs)\n[2] [blockchain-etl/bitcoin-etl](https://github.com/blockchain-etl/bitcoin-etl)", "tables/outputs": "The `outputs` table contains records of all transaction outputs within the Bitcoin blockchain. Each row in this table represents a single output from a Bitcoin transaction, detailing the amount transferred, the destination addresses, and other script-related information. This table is crucial for understanding the flow of Bitcoin and analyzing transaction patterns, especially when linked with the `[transactions](/tables/transactions.md)` and `[inputs](/tables/inputs.md)` tables.\n\n# Schema\n\n- `transaction_hash`: The hash of the transaction this output belongs to.\n- `block_hash`: The hash of the block containing this transaction.\n- `block_number`: The number of the block containing this transaction.\n- `block_timestamp`: The timestamp of the block containing this transaction.\n- `index`: The zero-based index of this output within its transaction.\n- `script_asm`: The script in assembly format.\n- `script_hex`: The script in hexadecimal format.\n- `required_signatures`: The number of signatures required to spend this output.\n- `type`: The type of the output script.\n- `addresses`: (REPEATED) Array of destination addresses for this output.\n- `value`: The value of the output in satoshis.\n\n# Common query patterns\n\n```sql\nSELECT\n  t.*\nFROM\n  `bigquery-public-data.crypto_bitcoin.outputs` AS t\nWHERE\n  t.transaction_hash = 'some_transaction_hash'\n```\n\n```sql\nSELECT\n  SUM(t.value) AS total_output_value\nFROM\n  `bigquery-public-data.crypto_bitcoin.outputs` AS t\nWHERE\n  t.block_number = 123456\n```\n\n```sql\nSELECT\n  t.type,\n  COUNT(*) AS output_count\nFROM\n  `bigquery-public-data.crypto_bitcoin.outputs` AS t\nWHERE\n  t.block_timestamp BETWEEN TIMESTAMP('2023-01-01') AND TIMESTAMP('2023-01-31')\nGROUP BY\n  t.type\nORDER BY\n  output_count DESC\n```\n\n# Citations\n\n[1] [Outputs Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/outputs)\n[2] [Bitcoin ETL on GitHub](https://github.com/blockchain-etl/bitcoin-etl)", "tables/transactions": "The `transactions` table in the [crypto_bitcoin](/datasets/crypto_bitcoin.md) dataset provides a complete record of every transaction ever processed on the Bitcoin blockchain. Each row represents a single transaction, offering granular details such as its hash, size, associated [block](/tables/blocks.md) information (hash, number, timestamp), and the total input and output values. Importantly, it includes detailed arrays for both [inputs](/tables/inputs.md) and [outputs](/tables/outputs.md), each specifying spent transaction details, script information, involved addresses, and values. This table is essential for in-depth analysis of transaction flows, tracing funds, and understanding the economic activity within the Bitcoin network. The grain is one row per transaction, with data spanning the entire history of the Bitcoin blockchain, partitioned by `block_timestamp_month`.\n\n# Schema\n- `hash` STRING REQUIRED: The hash of this transaction\n- `size` INTEGER: The size of this transaction in bytes\n- `virtual_size` INTEGER: The virtual transaction size (differs from size for witness transactions)\n- `version` INTEGER: Protocol version specified in block which contained this transaction\n- `lock_time` INTEGER: Earliest time that miners can include the transaction in their hashing of the Merkle root to attach it in the latest block of the blockchain\n- `block_hash` STRING REQUIRED: Hash of the block which contains this transaction\n- `block_number` INTEGER REQUIRED: Number of the block which contains this transaction\n- `block_timestamp` TIMESTAMP REQUIRED: Timestamp of the block which contains this transaction\n- `block_timestamp_month` DATE REQUIRED: Month of the block which contains this transaction\n- `input_count` INTEGER: The number of inputs in the transaction\n- `output_count` INTEGER: The number of outputs in the transaction\n- `input_value` NUMERIC: Total value of inputs in the transaction\n- `output_value` NUMERIC: Total value of outputs in the transaction\n- `is_coinbase` BOOLEAN: True if this transaction is a coinbase transaction\n- `fee` NUMERIC: The fee paid by this transaction\n- `inputs` RECORD REPEATED: Transaction inputs\n  - `index` INTEGER REQUIRED: 0-indexed number of an input within a transaction\n  - `spent_transaction_hash` STRING: The hash of the transaction which contains the output that this input spends\n  - `spent_output_index` INTEGER: The index of the output this input spends\n  - `script_asm` STRING: Symbolic representation of the bitcoin's script language op-codes\n  - `script_hex` STRING: Hexadecimal representation of the bitcoin's script language op-codes\n  - `sequence` INTEGER: A number intended to allow unconfirmed time-locked transactions to be updated before being finalized\n  - `required_signatures` INTEGER: The number of signatures required to authorize the spent output\n  - `type` STRING: The address type of the spent output\n  - `addresses` STRING REPEATED: Addresses which own the spent output\n  - `value` NUMERIC: The value in base currency attached to the spent output\n- `outputs` RECORD REPEATED: Transaction outputs\n  - `index` INTEGER REQUIRED: 0-indexed number of an output within a transaction\n  - `script_asm` STRING: Symbolic representation of the bitcoin's script language op-codes\n  - `script_hex` STRING: Hexadecimal representation of the bitcoin's script language op-codes\n  - `required_signatures` INTEGER: The number of signatures required to authorize spending of this output\n  - `type` STRING: The address type of the output\n  - `addresses` STRING REPEATED: Addresses which own this output\n  - `value` NUMERIC: The value in base currency attached to this output\n\n# Common query patterns\n```sql\n-- Get the total number of transactions per day\nSELECT\n    DATE(block_timestamp) AS transaction_date,\n    COUNT(hash) AS transaction_count\nFROM\n    `bigquery-public-data.crypto_bitcoin.transactions`\nWHERE\n    block_timestamp BETWEEN '2023-01-01' AND '2023-01-31'\nGROUP BY\n    transaction_date\nORDER BY\n    transaction_date DESC;\n```\n```sql\n-- Find transactions involving a specific address as an output\nSELECT\n    t.hash AS transaction_hash,\n    t.block_timestamp\nFROM\n    `bigquery-public-data.crypto_bitcoin.transactions` AS t,\n    UNNEST(t.outputs) AS output\nWHERE\n    '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa' IN UNNEST(output.addresses)\nLIMIT 10;\n```\n```sql\n-- Calculate the total fees collected in a given month\nSELECT\n    FORMAT_TIMESTAMP('%Y-%m', block_timestamp) AS transaction_month,\n    SUM(fee) AS total_fee\nFROM\n    `bigquery-public-data.crypto_bitcoin.transactions`\nWHERE\n    block_timestamp BETWEEN '2023-01-01' AND '2023-01-31'\nGROUP BY\n    transaction_month;\n```\n```sql\n-- Find duplicate transactions (anomaly detection)\nSELECT\n\u00a0\u00a0\u00a0*\nFROM (\n\u00a0SELECT\n\u00a0\u00a0\u00a0hash,\n\u00a0\u00a0\u00a0COUNT(hash) AS dup_transaction_count\n\u00a0FROM\n\u00a0\u00a0\u00a0`bigquery-public-data.crypto_bitcoin.transactions`\n\u00a0GROUP BY\n\u00a0\u00a0\u00a0hash)\nWHERE\n\u00a0dup_transaction_count > 1\n```\n\n# Citations\n[1] [Bitcoin Transactions](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/crypto_bitcoin/tables/transactions)\n[2] [Bitcoin ETL](https://github.com/blockchain-etl/bitcoin-etl)\n[3] [Bitcoin in BigQuery: blockchain analytics on public data](https://cloud.google.com/blog/products/gcp/bitcoin-in-bigquery-blockchain-analytics-on-public-data)"}, "types": ["BigQuery Dataset", "BigQuery Table"], "palette": {"BigQuery Dataset": "#8b5cf6", "BigQuery Table": "#3b82f6", "Reference": "#10b981"}};
</script>
<script>
(function () {
  const bundle = window.BUNDLE;
  const bundleName = window.BUNDLE_NAME;
  document.title = `${bundleName} — OKF Viewer`;
  document.getElementById("bundle-name").textContent = bundleName;

  // Populate type filter
  const typeSelect = document.getElementById("filter-type");
  for (const t of bundle.types) {
    const opt = document.createElement("option");
    opt.value = t;
    opt.textContent = t;
    typeSelect.appendChild(opt);
  }

  // Build reverse-link index for backlinks
  const backlinks = {};
  for (const edge of bundle.edges) {
    const { source, target } = edge.data;
    (backlinks[target] ||= []).push(source);
  }

  // Look up node label/type by id
  const nodeIndex = {};
  for (const n of bundle.nodes) nodeIndex[n.data.id] = n.data;

  const cy = cytoscape({
    container: document.getElementById("graph"),
    elements: [...bundle.nodes, ...bundle.edges],
    style: [
      {
        selector: "node",
        style: {
          "background-color": "data(color)",
          "label": "data(label)",
          "color": "#0f172a",
          "font-size": 11,
          "text-valign": "bottom",
          "text-margin-y": 4,
          "text-wrap": "wrap",
          "text-max-width": 120,
          "width": "data(size)",
          "height": "data(size)",
          "border-width": 1,
          "border-color": "#0f172a",
        },
      },
      {
        selector: "node:selected",
        style: {
          "border-width": 3,
          "border-color": "#f59e0b",
        },
      },
      {
        selector: "edge",
        style: {
          "width": 1.5,
          "line-color": "#cbd5e1",
          "target-arrow-color": "#cbd5e1",
          "target-arrow-shape": "triangle",
          "curve-style": "bezier",
          "arrow-scale": 0.9,
        },
      },
      {
        selector: "edge:selected",
        style: {
          "line-color": "#f59e0b",
          "target-arrow-color": "#f59e0b",
          "width": 2.5,
        },
      },
      {
        selector: ".dim",
        style: { "opacity": 0.15 },
      },
    ],
    layout: { name: "cose", animate: false, padding: 30 },
    wheelSensitivity: 0.2,
  });

  cy.on("tap", "node", (evt) => showDetail(evt.target.id()));
  cy.on("tap", (evt) => {
    if (evt.target === cy) clearSelection();
  });

  document.getElementById("layout").addEventListener("change", (e) => {
    cy.layout({ name: e.target.value, animate: false, padding: 30 }).run();
  });

  document.getElementById("reset").addEventListener("click", () => {
    cy.fit(null, 30);
    clearSelection();
  });

  document.getElementById("search").addEventListener("input", (e) => {
    const q = e.target.value.trim().toLowerCase();
    if (!q) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      const d = n.data();
      const hay =
        (d.label || "").toLowerCase() + " " +
        d.id.toLowerCase() + " " +
        (d.tags || []).join(" ").toLowerCase();
      n.toggleClass("dim", !hay.includes(q));
    });
    cy.edges().forEach((edge) => {
      const src = edge.source();
      const tgt = edge.target();
      edge.toggleClass("dim", src.hasClass("dim") || tgt.hasClass("dim"));
    });
  });

  document.getElementById("filter-type").addEventListener("change", (e) => {
    const t = e.target.value;
    if (!t) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      n.toggleClass("dim", n.data("type") !== t);
    });
    cy.edges().forEach((edge) => {
      edge.toggleClass("dim", edge.source().hasClass("dim") || edge.target().hasClass("dim"));
    });
  });

  function clearSelection() {
    cy.elements().unselect();
    document.getElementById("detail-empty").hidden = false;
    document.getElementById("detail-content").hidden = true;
  }

  function showDetail(conceptId) {
    const data = nodeIndex[conceptId];
    if (!data) return;
    cy.elements().unselect();
    const node = cy.getElementById(conceptId);
    if (node) node.select();

    document.getElementById("detail-empty").hidden = true;
    const content = document.getElementById("detail-content");
    content.hidden = false;

    const chip = document.getElementById("detail-type");
    chip.textContent = data.type;
    chip.style.background = data.color;

    document.getElementById("detail-title").textContent = data.label;
    document.getElementById("detail-id").textContent = conceptId;
    document.getElementById("detail-description").textContent = data.description || "—";

    const resourceEl = document.getElementById("detail-resource");
    resourceEl.innerHTML = "";
    if (data.resource) {
      const a = document.createElement("a");
      a.href = data.resource;
      a.textContent = data.resource;
      a.target = "_blank";
      a.rel = "noopener";
      a.className = "external";
      resourceEl.appendChild(a);
    } else {
      resourceEl.textContent = "—";
    }

    const tagsEl = document.getElementById("detail-tags");
    tagsEl.innerHTML = "";
    if (data.tags && data.tags.length) {
      for (const t of data.tags) {
        const span = document.createElement("span");
        span.className = "tag";
        span.textContent = t;
        tagsEl.appendChild(span);
      }
    } else {
      tagsEl.textContent = "—";
    }

    const body = bundle.bodies[conceptId] || "";
    const html = marked.parse(body, { breaks: false, gfm: true });
    const bodyEl = document.getElementById("detail-body");
    bodyEl.innerHTML = html;
    rewriteInternalLinks(bodyEl);

    const bl = backlinks[conceptId] || [];
    const blSection = document.getElementById("detail-backlinks");
    const blList = document.getElementById("backlinks-list");
    blList.innerHTML = "";
    if (bl.length) {
      blSection.hidden = false;
      for (const src of bl) {
        const li = document.createElement("li");
        const a = document.createElement("a");
        a.textContent = nodeIndex[src]?.label || src;
        a.dataset.target = src;
        a.addEventListener("click", () => showDetail(src));
        li.appendChild(a);
        const muted = document.createElement("span");
        muted.className = "muted";
        muted.textContent = ` (${src})`;
        li.appendChild(muted);
        blList.appendChild(li);
      }
    } else {
      blSection.hidden = true;
    }

    cy.animate({ center: { eles: node }, zoom: Math.max(cy.zoom(), 1.0) }, { duration: 200 });
  }

  function rewriteInternalLinks(root) {
    root.querySelectorAll("a[href]").forEach((a) => {
      const href = a.getAttribute("href");
      if (!href) return;
      if (href.startsWith("/") && href.endsWith(".md")) {
        const target = href.slice(1, -3);
        if (nodeIndex[target]) {
          a.className = "internal";
          a.setAttribute("href", "javascript:void(0)");
          a.addEventListener("click", (e) => {
            e.preventDefault();
            showDetail(target);
          });
          return;
        }
      }
      a.className = "external";
      a.setAttribute("target", "_blank");
      a.setAttribute("rel", "noopener");
    });
  }

  // Auto-show the first node (a dataset if available, else first concept)
  const initial =
    bundle.nodes.find((n) => n.data.type === "BigQuery Dataset") ||
    bundle.nodes[0];
  if (initial) showDetail(initial.data.id);
})();

</script>
</body>
</html>


================================================
FILE: okf\bundles\ga4\datasets\ga4_obfuscated_sample_ecommerce.md
================================================
---
type: BigQuery Dataset
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce
title: BigQuery sample dataset for Google Analytics ecommerce web implementation
description: A sample of obfuscated Google Analytics BigQuery event export data for
  three months from the Google Merchandise Store is available as a public dataset
  in BigQuery.
tags:
- ecommerce
- web analytics
- Google Analytics
- BigQuery
- public dataset
timestamp: '2026-05-28T22:49:59+00:00'
---

# Overview
The `ga4_obfuscated_sample_ecommerce` dataset contains obfuscated Google Analytics BigQuery event export data for three months (November 2020 to January 2021) from the Google Merchandise Store. This public dataset is available in BigQuery and emulates a real-world dataset.

# Pre-requisites
To work with this dataset, you need access to a Google Cloud project with the BigQuery API enabled. You can use BigQuery Sandbox mode or the Free usage tier for exploration and sample queries.

# Limitations
The dataset contains obfuscated data with placeholder values like `<Other>`, `NULL`, and `''`. Due to obfuscation, the internal consistency of the dataset might be somewhat limited. It cannot be compared to the Google Analytics Demo Account.

# Using the dataset
You can access the `ga4_obfuscated_sample_ecommerce` dataset via the BigQuery UI in the Cloud Console.

## Sample Query
The following query shows the number of unique events, users, and days in the dataset:

```sql
SELECT
  COUNT(*) AS event_count,
  COUNT(DISTINCT user_pseudo_id) AS user_count,
  COUNT(DISTINCT event_date) AS day_count
FROM `bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*`
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: okf\bundles\ga4\datasets\index.md
================================================
# BigQuery Dataset

* [BigQuery sample dataset for Google Analytics ecommerce web implementation](ga4_obfuscated_sample_ecommerce.md) - A sample of obfuscated Google Analytics BigQuery event export data for three months from the Google Merchandise Store is available as a public dataset in BigQuery.


================================================
FILE: okf\bundles\ga4\index.md
================================================
# Subdirectories

* [datasets](datasets/index.md) - A sample of obfuscated Google Analytics BigQuery event export data for three months from the Google Merchandise Store is available as a public dataset in BigQuery.
* [references](references/index.md) - This directory contains specifications for data joins and definitions for user activity and purchase metrics.
* [tables](tables/index.md) - Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.


================================================
FILE: okf\bundles\ga4\references\index.md
================================================
# Subdirectories

* [joins](joins/index.md) - Join Google Analytics event data with Google Ads click data.
* [metrics](metrics/index.md) - This directory contains definitions for metrics related to user activity, events, and purchase behavior.


================================================
FILE: okf\bundles\ga4\references\joins\events___ads_clickstats.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Join Google Analytics Events to Google Ads Clicks
description: Join Google Analytics event data with Google Ads click data.
tags:
- join
- Google Ads
timestamp: '2026-05-28T22:51:46+00:00'
---

Join Google Analytics event data with Google Ads click data.

```sql
GA_EVENTS.collected_traffic_source.gclid = ADS_CLICKS.gclid
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: okf\bundles\ga4\references\joins\index.md
================================================
# Reference

* [Join Google Analytics Events to Google Ads Clicks](events___ads_clickstats.md) - Join Google Analytics event data with Google Ads click data.


================================================
FILE: okf\bundles\ga4\references\metrics\avg_pageviews.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Average Pageviews
description: The average number of pageviews per user.
tags:
- metric
timestamp: '2026-05-28T22:51:43+00:00'
---

The average number of pageviews per user.

```sql
SUM(page_view_count) / COUNT(*)
-- where page_view_count is COUNTIF(event_name = 'page_view') per user
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: okf\bundles\ga4\references\metrics\avg_spend_per_purchase_session_by_user.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/advanced-queries
title: Average Spend Per Purchase Session By User
description: The average amount of money spent per purchase session for each individual
  user.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:52:29+00:00'
---

The average amount of money spent per purchase session for each individual user.

```sql
AVG(total_session_spend)
-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id
```

# Citations
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: okf\bundles\ga4\references\metrics\avg_transactions_per_purchaser.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Average Transactions Per Purchaser
description: The average number of transactions made by purchasers.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:51:41+00:00'
---

The average number of transactions made by purchasers.

```sql
COUNT(*) / COUNT(DISTINCT user_pseudo_id)
-- for events where event_name IN ('in_app_purchase', 'purchase')
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: okf\bundles\ga4\references\metrics\day_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: Day Count
description: Total number of unique days.
tags:
- metric
timestamp: '2026-05-28T22:50:10+00:00'
---

Total number of unique days.

```sql
COUNT(DISTINCT event_date)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: okf\bundles\ga4\references\metrics\event_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: Event Count
description: Total number of events.
tags:
- metric
timestamp: '2026-05-28T22:50:07+00:00'
---

Total number of events.

```sql
COUNT(*)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: okf\bundles\ga4\references\metrics\index.md
================================================
# Reference

* [Average Pageviews](avg_pageviews.md) - The average number of pageviews per user.
* [Average Spend Per Purchase Session By User](avg_spend_per_purchase_session_by_user.md) - The average amount of money spent per purchase session for each individual user.
* [Average Transactions Per Purchaser](avg_transactions_per_purchaser.md) - The average number of transactions made by purchasers.
* [Day Count](day_count.md) - Total number of unique days.
* [Event Count](event_count.md) - Total number of events.
* [New User Count](new_user_count.md) - The number of unique users who triggered a first_visit or first_open event.
* [Overall Average Spend Per Purchase Session](overall_avg_spend_per_purchase_session.md) - The overall average amount spent across all unique purchase sessions.
* [User Count](user_count.md) - Total number of unique users.


================================================
FILE: okf\bundles\ga4\references\metrics\new_user_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: New User Count
description: The number of unique users who triggered a first_visit or first_open
  event.
tags:
- metric
timestamp: '2026-05-28T22:51:38+00:00'
---

The number of unique users who triggered a `first_visit` or `first_open` event.

```sql
SUM(is_new_user)
-- where is_new_user is MAX(IF(event_name IN ('first_visit', 'first_open'), 1, 0)) grouped by user_pseudo_id
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: okf\bundles\ga4\references\metrics\overall_avg_spend_per_purchase_session.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/advanced-queries
title: Overall Average Spend Per Purchase Session
description: The overall average amount spent across all unique purchase sessions.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:52:32+00:00'
---

The overall average amount spent across all unique purchase sessions.

```sql
AVG(total_session_spend)
-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id
```

# Citations
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: okf\bundles\ga4\references\metrics\user_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: User Count
description: Total number of unique users.
tags:
- metric
timestamp: '2026-05-28T22:50:09+00:00'
---

Total number of unique users.

```sql
COUNT(DISTINCT user_pseudo_id)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: okf\bundles\ga4\tables\events_.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce/tables/events_*
title: Events table (Google Analytics BigQuery Export)
description: Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce`
  dataset.
tags:
- events
- Google Analytics
- BigQuery
- ecommerce
- schema
- basic queries
- advanced queries
timestamp: '2026-05-28T22:53:05+00:00'
---

# Overview
The `events_` table is a sharded BigQuery table containing Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.

# Metrics
- [Event Count](../references/metrics/event_count.md) — Total number of events.
- [User Count](../references/metrics/user_count.md) — Total number of unique users.
- [Day Count](../references/metrics/day_count.md) — Total number of unique days.
- [New User Count](../references/metrics/new_user_count.md) — The number of unique users who triggered a first_visit or first_open event.
- [Average Transactions Per Purchaser](../references/metrics/avg_transactions_per_purchaser.md) — The average number of transactions made by purchasers.
- [Average Pageviews](../references/metrics/avg_pageviews.md) — The average number of pageviews per user.
- [Average Spend Per Purchase Session By User](../references/metrics/avg_spend_per_purchase_session_by_user.md) — The average amount of money spent per purchase session for each individual user.
- [Overall Average Spend Per Purchase Session](../references/metrics/overall_avg_spend_per_purchase_session.md) — The overall average amount spent across all unique purchase sessions.

# Schema
The `events_YYYYMMDD` table, created daily, and `events_intraday_YYYYMMDD` (for streaming export) contain the following fields:

## event
The event fields contain information that uniquely identifies an event.

- `batch_event_index` (INTEGER): A number indicating the sequential order of each event within a batch based on their order of occurrence on the device.
- `batch_ordering_id` (INTEGER): A monotonically increasing number that is incremented each time a network request is sent from a given page.
- `batch_page_id` (INTEGER): A sequential number assigned to a page that increases for each subsequent page within an engagement.
- `event_date` (STRING): The date when the event was logged (YYYYMMDD format in the registered timezone of your app).
- `event_timestamp` (INTEGER): The time (in microseconds, UTC) when the event was received by Google Analytics. Multiple events can share the same timestamp if sent in the same request.
- `event_previous_timestamp` (INTEGER): The time (in microseconds, UTC) when the previous event happened.
- `event_name` (STRING): The name of the event.
- `event_value_in_usd` (FLOAT): The currency-converted value (in USD) of the event's "value" parameter.
- `event_bundle_sequence_id` (INTEGER): The sequential ID of the bundle in which these events were uploaded.
- `event_server_timestamp_offset` (INTEGER): Timestamp offset between collection time and upload time in micros.

### event_params RECORD
The `event_params` RECORD can store campaign-level and contextual event parameters as well as any user-defined event parameters. The `event_params` RECORD is repeated for each key that is associated with an event.

- `event_params.key` (STRING): The name of the event parameter.
- `event_params.value` (RECORD): A record containing the event parameter's value.
    - `event_params.value.string_value` (STRING): If the event parameter is represented by a string, such as a URL or campaign name, it is populated in this field.
    - `event_params.value.int_value` (INTEGER): If the event parameter is represented by an integer, it is populated in this field.
    - `event_params.value.double_value` (FLOAT): If the event parameter is represented by a double value, it is populated in this field.
    - `event_params.value.float_value` (FLOAT): If the event parameter is represented by a floating point value, it is populated in this field. This field is not currently in use.
- `event_params.key = 'page_location'` (STRING): The URL of the page viewed.
- `event_params.key = 'page_title'` (STRING): The title of the page viewed.
- `event_params.key = 'ga_session_id'` (INTEGER): The ID of the Google Analytics session.
- `event_params.key = 'ga_session_number'` (INTEGER): The sequential number of the session for a user.

### item_params RECORD
The `item_params` RECORD can store item parameters as well as any user-defined item parameters.

- `item_params.key` (STRING): The name of the item parameter.
- `item_params.value` (RECORD): A record containing the item parameter’s value.
    - `item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.
    - `item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.
    - `item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.
    - `item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.

## user
The user fields contain information that uniquely identifies the user associated with the event.

- `is_active_user` (BOOLEAN): Whether the user was active (True) or inactive (False) at any point in the calendar day. This field is only populated in the daily tables (`events_YYYYMMDD`).
- `user_id` (STRING): The unique ID assigned to a user.
- `user_pseudo_id` (STRING): The pseudonymous id (e.g., app instance ID) for the user. A unique identifier that is assigned to a user when they first open the app or visit the site.
- `user_first_touch_timestamp` (INTEGER): The time (in microseconds) at which the user first opened the app or visited the site.

### privacy_info fields
The `privacy_info` fields contain information based on the consent status of a user when consent mode is enabled.

- `privacy_info.ads_storage` (STRING): Whether ad targeting is enabled for a user. Possible values: Yes, No, Unset
- `privacy_info.analytics_storage` (STRING): Whether Analytics storage is enabled for the user. Possible values: Yes, No, Unset
- `privacy_info.uses_transient_token` (STRING): Whether a web user has denied Analytics storage and the developer has enabled measurement without cookies based on transient tokens in server data. Possible values: Yes, No, Unset

### user_properties RECORD
The `user_properties` RECORD contains any user properties that you have set. It is repeated for each key that is associated with a user.

- `user_properties.key` (STRING): The name of the user property.
- `user_properties.value` (RECORD): A record for the user property value.
    - `user_properties.value.string_value` (STRING): The string value of the user property.
    - `user_properties.value.int_value` (INTEGER): The integer value of the user property.
    - `user_properties.value.double_value` (FLOAT): The double value of the user property.
    - `user_properties.value.float_value` (FLOAT): This field is currently unused.
    - `user_properties.value.set_timestamp_micros` (INTEGER): The time (in microseconds) at which the user property was last set.

### user_ltv RECORD
The `user_ltv` RECORD contains Lifetime Value information about the user. This RECORD is not populated in intraday tables.

- `user_ltv.revenue` (FLOAT): The Lifetime Value (revenue) of the user. This field is not populated in intraday tables.
- `user_ltv.currency` (STRING): The Lifetime Value (currency) of the user. This field is not populated in intraday tables.

## device
The device RECORD contains information about the device from which the event originated.

- `device.category` (STRING): The device category (mobile, tablet, desktop).
- `device.mobile_brand_name` (STRING): The device brand name.
- `device.mobile_model_name` (STRING): The device model name.
- `device.mobile_marketing_name` (STRING): The device marketing name.
- `device.mobile_os_hardware_model` (STRING): The device model information retrieved directly from the operating system.
- `device.operating_system` (STRING): The operating system of the device.
- `device.operating_system_version` (STRING): The OS version.
- `device.vendor_id` (STRING): IDFV (present only if IDFA is not collected).
- `device.advertising_id` (STRING): Advertising ID/IDFA.
- `device.language` (STRING): The OS language.
- `device.time_zone_offset_seconds` (INTEGER): The offset from GMT in seconds.
- `device.is_limited_ad_tracking` (BOOLEAN): The device's Limit Ad Tracking setting. On iOS14+, returns false if the IDFA is non-zero.
- `device.web_info.browser` (STRING): The browser in which the user viewed content.
- `device.web_info.browser_version` (STRING): The version of the browser in which the user viewed content.
- `device.web_info.hostname` (STRING): The hostname associated with the logged event.

## geo
The geo RECORD contains information about the geographic location where the event was initiated.

- `geo.continent` (STRING): The continent from which events were reported, based on IP address.
- `geo.sub_continent` (STRING): The subcontinent from which events were reported, based on IP address.
- `geo.country` (STRING): The country from which events were reported, based on IP address.
- `geo.region` (STRING): The region from which events were reported, based on IP address.
- `geo.metro` (STRING): The metro from which events were reported, based on IP address.
- `geo.city` (STRING): The city from which events were reported, based on IP address.

## app_info
The `app_info` RECORD contains information about the app in which the event was initiated.

- `app_info.id` (STRING): The package name or bundle ID of the app.
- `app_info.firebase_app_id` (STRING): The Firebase App ID associated with the app.
- `app_info.install_source` (STRING): The store that installed the app.
- `app_info.version` (STRING): The app's versionName (Android) or short bundle version.

## collected_traffic_source
The `collected_traffic_source` RECORD contains the traffic source data that was present within the events that were collected.

- `manual_campaign_id` (STRING): The manual campaign id (utm_id) that was collected with the event.
- `manual_campaign_name` (STRING): The manual campaign name (utm_campaign) that was collected with the event.
- `manual_source` (STRING): The manual campaign source (utm_source) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.
- `manual_medium` (STRING): The manual campaign medium (utm_medium) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.
- `manual_term` (STRING): The manual campaign keyword/term (utm_term) that was collected with the event.
- `manual_content` (STRING): The additional manual campaign metadata (utm_content) that was collected with the event.
- `manual_creative_format` (STRING): The manual campaign creative format (utm_creative_format) that was collected with the event.
- `manual_marketing_tactic` (STRING): The manual campaign marketing tactic (utm_marketing_tactic) that was collected with the event.
- `manual_source_platform` (STRING): The manual campaign source platform (utm_source_platform) that was collected with the event.
- `gclid` (STRING): The Google click identifier that was collected with the event.
- `dclid` (STRING): The DoubleClick Click Identifier for Display and Video 360 and Campaign Manager 360 that was collected with the event.
- `srsltid` (STRING): The Google Merchant Center identifier that was collected with the event.

## session_traffic_source_last_click
The `session_traffic_source_last_click` RECORD contains the last-click attributed session traffic source data across Google ads and manual contexts, where available.

- `session_traffic_source_last_click.manual_campaign.campaign_id` (STRING): The ID of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.campaign_name` (STRING): The name of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.medium` (STRING): The medium of the last clicked manual campaign (e.g., paid search, organic search, email).
- `session_traffic_source_last_click.manual_campaign.term` (STRING): The keyword/search term of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.content` (STRING): Additional metadata of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.source_platform` (STRING): The platform of the last clicked manual campaign (e.g., search engine, social media).
- `session_traffic_source_last_click.manual_campaign.source` (STRING): The specific source within the platform of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.creative_format` (STRING): The format of the creative of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked manual campaign.
- `session_traffic_source_last_click.google_ads_campaign.customer_id` (STRING): The customer ID associated with the Google Ads account.
- `session_traffic_source_last_click.google_ads_campaign.account_name` (STRING): The name of the Google Ads account.
- `session_traffic_source_last_click.google_ads_campaign.campaign_id` (STRING): The ID of the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.campaign_name` (STRING): The name of the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.ad_group_id` (STRING): The ID of the ad group within the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.ad_group_name` (STRING): The name of the ad group within the Google Ads campaign.
- `session_traffic_source_last_click.cross_channel_campaign.campaign_name` (STRING): The name of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.campaign_id` (STRING): The ID of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.source_platform` (STRING): The platform of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.source` (STRING): The specific source within the platform of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.medium` (STRING): The medium of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.sa360_campaign.campaign_name` (STRING): The name of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.source` (STRING): The specific source within the platform of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.medium` (STRING): The medium of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.ad_group_id` (STRING): The ID of the ad group within the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.ad_group_name` (STRING): The name of the ad group within the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.campaign_id` (STRING): The ID of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.creative_format` (STRING): The format of the creative of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.engine_account_name` (STRING): The name of the SA360 engine account.
- `session_traffic_source_last_click.sa360_campaign.engine_account_type` (STRING): The type of engine account containing the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.manager_account_name` (STRING): The name of the SA360 manager account.
- `session_traffic_source_last_click.dv360_campaign.advertiser_id` (STRING): The ID of the DV360 advertiser.
- `session_traffic_source_last_click.dv360_campaign.advertiser_name` (STRING): The name of the DV360 advertiser.
- `session_traffic_source_last_click.dv360_campaign.campaign_id` (STRING): The ID of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.campaign_name` (STRING): The name of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_id` (STRING): The ID of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_format` (STRING): The format of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_name` (STRING): The name of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.exchange_id` (STRING): The exchange ID of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.exchange_name` (STRING): The exchange name of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.insertion_order_id` (STRING): The ID of the insertion order in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.insertion_order_name` (STRING): The name of the insertion order in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.line_item_id` (STRING): The ID of the line item in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.line_item_name` (STRING): The name of the line item in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.partner_id` (STRING): The ID of the DV360 partner.
- `session_traffic_source_last_click.dv360_campaign.partner_name` (STRING): The name of the DV360 partner.
- `session_traffic_source_last_click.dv360_campaign.source` (STRING): The specific source within the platform of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.medium` (STRING): The medium of the last clicked DV360 campaign.
- `session_traffic_source_last_click.cm360_campaign.account_id` (STRING): The ID of the CM360 account.
- `session_traffic_source_last_click.cm360_campaign.account_name` (STRING): The name of the CM360 account.
- `session_traffic_source_last_click.cm360_campaign.advertiser_id` (STRING): The ID of the CM360 advertiser.
- `session_traffic_source_last_click.cm360_campaign.advertiser_name` (STRING): The name of the CM360 advertiser.
- `session_traffic_source_last_click.cm360_campaign.campaign_id` (STRING): The ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.campaign_name` (STRING): The name of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_id` (STRING): The ID of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_format` (STRING): The format of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_name` (STRING): The name of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_type` (STRING): The creative type of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_type_id` (STRING): The creative type ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_version` (STRING): The creative version of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_id` (STRING): The ID of the placement of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_cost_structure` (STRING): The placement cost structure of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_name` (STRING): The name of the placement of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.rendering_id` (STRING): The rendering ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.site_id` (STRING): The site ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.site_name` (STRING): The site name of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.source` (STRING): The specific source of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.medium` (STRING): The medium of the last clicked CM360 campaign.

## traffic_source
The `traffic_source` RECORD contains information about the traffic source that first acquired the user. This record is not populated in intraday tables.

- `traffic_source.name` (STRING): Name of the marketing campaign that first acquired the user. This field is not populated in intraday tables.
- `traffic_source.medium` (STRING): Name of the medium (paid search, organic search, email, etc.) that first acquired the user. This field is not populated in intraday tables.
- `traffic_source.source` (STRING): Name of the network that first acquired the user. This field is not populated in intraday tables.

## stream and platform
The stream and platform fields contain information about the stream and the app platform.

- `stream_id` (STRING): The numeric ID of the data stream from which the event originated.
- `platform` (STRING): The data stream platform (Web, IOS or Android) from which the event originated.

## ecommerce
This ecommerce RECORD contains information about any ecommerce events that have been setup on a website or app.

- `ecommerce.total_item_quantity` (INTEGER): Total number of items in this event, which is the sum of items.quantity.
- `ecommerce.purchase_revenue_in_usd` (FLOAT): Purchase revenue of this event, represented in USD with standard unit. Populated for purchase event only.
- `ecommerce.purchase_revenue` (FLOAT): Purchase revenue of this event, represented in local currency with standard unit. Populated for purchase event only.
- `ecommerce.refund_value_in_usd` (FLOAT): The amount of refund in this event, represented in USD with standard unit. Populated for refund event only.
- `ecommerce.refund_value` (FLOAT): The amount of refund in this event, represented in local currency with standard unit. Populated for refund event only.
- `ecommerce.shipping_value_in_usd` (FLOAT): The shipping cost in this event, represented in USD with standard unit.
- `ecommerce.shipping_value` (FLOAT): The shipping cost in this event, represented in local currency.
- `ecommerce.tax_value_in_usd` (FLOAT): The tax value in this event, represented in USD with standard unit.
- `ecommerce.tax_value` (FLOAT): The tax value in this event, represented in local currency with standard unit.
- `ecommerce.transaction_id` (STRING): The transaction ID of the ecommerce transaction.
- `ecommerce.unique_items` (INTEGER): The number of unique items in this event, based on item_id, item_name, and item_brand.

## items
The `items` RECORD contains information about items included in an event. It is repeated for each item.

- `items.item_id` (STRING): The ID of the item.
- `items.item_name` (STRING): The name of the item.
- `items.item_brand` (STRING): The brand of the item.
- `items.item_variant` (STRING): The variant of the item.
- `items.item_category` (STRING): The category of the item.
- `items.item_category2` (STRING): The sub category of the item.
- `items.item_category3` (STRING): The sub category of the item.
- `items.item_category4` (STRING): The sub category of the item.
- `items.item_category5` (STRING): The sub category of the item.
- `items.price_in_usd` (FLOAT): The price of the item, in USD with standard unit.
- `items.price` (FLOAT): The price of the item in local currency.
- `items.quantity` (INTEGER): The quantity of the item. Quantity set to 1 if not specified.
- `items.item_revenue_in_usd` (FLOAT): The revenue of this item, calculated as price_in_usd * quantity. It is populated for purchase events only, in USD with standard unit.
- `items.item_revenue` (FLOAT): The revenue of this item, calculated as price * quantity. It is populated for purchase events only, in local currency with standard unit.
- `items.item_refund_in_usd` (FLOAT): The refund value of this item, calculated as price_in_usd * quantity. It is populated for refund events only, in USD with standard unit.
- `items.item_refund` (FLOAT): The refund value of this item, calculated as price * quantity. It is populated for refund events only, in local currency with standard unit.
- `items.coupon` (STRING): Coupon code applied to this item.
- `items.affiliation` (STRING): A product affiliation to designate a supplying company or brick and mortar store location.
- `items.location_id` (STRING): The location associated with the item.
- `items.item_list_id` (STRING): The ID of the list in which the item was presented to the user.
- `items.item_list_name` (STRING): The name of the list in which the item was presented to the user.
- `items.item_list_index` (STRING): The position of the item in a list.
- `items.promotion_id` (STRING): The ID of a product promotion.
- `items.promotion_name` (STRING): The name of a product promotion.
- `items.creative_name` (STRING): The name of a creative used in a promotional spot.
- `items.creative_slot` (STRING): The name of a creative slot.

### item_params RECORD (nested under items)
The `item_params` RECORD stores the custom item parameters that you defined.

- `items.item_params.key` (STRING): The name of the item parameter.
- `items.item_params.value` (RECORD): A record containing the item parameter’s value.
    - `items.item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.
    - `items.item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.
    - `items.item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.
    - `items.item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.

## publisher (Early access only)
The publisher RECORD contains information about events sourced from a publisher integration related to the display of ads, that is, AdMob.

- `publisher` (RECORD): A record of publisher data coming from AdMob. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_revenue_in_usd` (FLOAT): Estimated ad revenue resulting from this event, represented in USD. Populated for ad impression events only. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_format` (STRING): Describes the way ads appeared and where they were located. Typical formats include ‘Interstitial’, ‘Banner’, ‘Rewarded’, and ‘Native advanced’. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_source_name` (STRING): The source network that served an ad. Typical sources include, ‘AdMob Network’, ‘Meta audience Network’, and ‘Mediated house ads’. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_unit_id` (STRING): The name you chose to describe this Ad unit. Ad units are containers that you place in your apps to show ads to users. This field is not populated in intraday tables and Fresh Daily BigQuery export.

# Joins
- [Google Analytics Events to Google Ads Clicks](../references/joins/events___ads_clickstats.md) — join on `collected_traffic_source.gclid` to attach Google Ads data to events.

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
- https://support.google.com/analytics/answer/7029846
- https://developers.google.com/analytics/bigquery/basic-queries
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: okf\bundles\ga4\tables\index.md
================================================
# BigQuery Table

* [Events table (Google Analytics BigQuery Export)](events_.md) - Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.


================================================
FILE: okf\bundles\ga4\viz.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>OKF Bundle Viewer</title>
<script src="https://cdn.jsdelivr.net/npm/cytoscape@3.28.1/dist/cytoscape.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script>
<style>
* { box-sizing: border-box; }
body {
  margin: 0;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
  font-size: 14px;
  color: #0f172a;
  background: #f8fafc;
  display: flex;
  flex-direction: column;
  height: 100vh;
}
header {
  display: flex;
  align-items: center;
  justify-content: space-between;
  padding: 10px 16px;
  background: #fff;
  border-bottom: 1px solid #e2e8f0;
  flex-shrink: 0;
}
.title strong { font-size: 16px; margin-right: 8px; }
.muted { color: #64748b; font-size: 12px; }
.controls { display: flex; gap: 8px; }
.controls input, .controls select, .controls button {
  font-size: 13px;
  padding: 5px 8px;
  border: 1px solid #cbd5e1;
  border-radius: 4px;
  background: #fff;
}
.controls input { width: 220px; }
.controls button { cursor: pointer; background: #f1f5f9; }
.controls button:hover { background: #e2e8f0; }

main {
  display: flex;
  flex: 1;
  min-height: 0;
}
#graph {
  flex: 1 1 60%;
  background: #fff;
  border-right: 1px solid #e2e8f0;
  min-width: 0;
  position: relative;
}
#detail {
  flex: 0 0 40%;
  overflow-y: auto;
  padding: 18px 22px;
  background: #fff;
}
#detail-empty {
  text-align: center;
  margin-top: 40px;
}

.detail-header { margin-bottom: 12px; }
.detail-header h1 {
  font-size: 18px;
  margin: 4px 0 2px;
  font-weight: 600;
}
.type-chip {
  display: inline-block;
  padding: 2px 8px;
  border-radius: 10px;
  font-size: 11px;
  font-weight: 600;
  color: #fff;
  background: #94a3b8;
  text-transform: uppercase;
  letter-spacing: 0.5px;
}
dl.frontmatter {
  display: grid;
  grid-template-columns: 90px 1fr;
  row-gap: 4px;
  column-gap: 12px;
  margin: 8px 0 12px;
  font-size: 13px;
}
dl.frontmatter dt {
  color: #64748b;
  font-weight: 500;
}
dl.frontmatter dd { margin: 0; }
dl.frontmatter a { color: #2563eb; word-break: break-all; }

.tag {
  display: inline-block;
  padding: 1px 6px;
  margin: 0 4px 2px 0;
  border-radius: 4px;
  background: #f1f5f9;
  color: #475569;
  font-size: 11px;
}

hr { border: none; border-top: 1px solid #e2e8f0; margin: 14px 0; }

#detail-body { font-size: 13px; line-height: 1.55; }
#detail-body h1 {
  font-size: 16px; margin: 18px 0 6px;
  padding-bottom: 4px; border-bottom: 1px solid #e2e8f0;
}
#detail-body h2 { font-size: 14px; margin: 14px 0 4px; }
#detail-body h3 { font-size: 13px; margin: 12px 0 4px; }
#detail-body p { margin: 6px 0; }
#detail-body code {
  background: #f1f5f9;
  padding: 1px 4px;
  border-radius: 3px;
  font-size: 12px;
  font-family: ui-monospace, "SF Mono", Consolas, monospace;
}
#detail-body pre {
  background: #0f172a;
  color: #e2e8f0;
  padding: 10px 12px;
  border-radius: 6px;
  overflow-x: auto;
  font-size: 12px;
}
#detail-body pre code { background: transparent; color: inherit; padding: 0; }
#detail-body ul, #detail-body ol { padding-left: 22px; margin: 6px 0; }
#detail-body li { margin: 2px 0; }
#detail-body table { border-collapse: collapse; margin: 8px 0; }
#detail-body th, #detail-body td {
  border: 1px solid #e2e8f0; padding: 4px 8px; font-size: 12px;
}
#detail-body a.internal { color: #2563eb; cursor: pointer; }
#detail-body a.external { color: #2563eb; }

#detail-backlinks { margin-top: 18px; }
#detail-backlinks h2 { font-size: 13px; color: #64748b; margin-bottom: 6px; }
#detail-backlinks ul { padding-left: 18px; }
#detail-backlinks a { color: #2563eb; cursor: pointer; }

</style>
</head>
<body>
<header>
  <div class="title">
    <strong id="bundle-name"></strong>
    <span class="muted">OKF bundle</span>
  </div>
  <div class="controls">
    <input id="search" type="search" placeholder="Search title / id / tag">
    <select id="filter-type">
      <option value="">All types</option>
    </select>
    <select id="layout">
      <option value="cose">cose (force)</option>
      <option value="concentric">concentric</option>
      <option value="breadthfirst">breadth-first</option>
      <option value="circle">circle</option>
      <option value="grid">grid</option>
    </select>
    <button id="reset">Reset view</button>
  </div>
</header>

<main>
  <section id="graph"></section>
  <section id="detail">
    <div id="detail-empty" class="muted">Click a node to see its details.</div>
    <article id="detail-content" hidden>
      <header class="detail-header">
        <span class="type-chip" id="detail-type"></span>
        <h1 id="detail-title"></h1>
        <div class="muted" id="detail-id"></div>
      </header>
      <dl class="frontmatter">
        <dt>Description</dt><dd id="detail-description"></dd>
        <dt>Resource</dt><dd id="detail-resource"></dd>
        <dt>Tags</dt><dd id="detail-tags"></dd>
      </dl>
      <hr>
      <div id="detail-body"></div>
      <section id="detail-backlinks" hidden>
        <h2>Cited by</h2>
        <ul id="backlinks-list"></ul>
      </section>
    </article>
  </section>
</main>

<script>
window.BUNDLE_NAME = "ga4";
window.BUNDLE = {"nodes": [{"data": {"id": "datasets/ga4_obfuscated_sample_ecommerce", "label": "BigQuery sample dataset for Google Analytics ecommerce web implementation", "type": "BigQuery Dataset", "description": "A sample of obfuscated Google Analytics BigQuery event export data for three months from the Google Merchandise Store is available as a public dataset in BigQuery.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce", "tags": ["ecommerce", "web analytics", "Google Analytics", "BigQuery", "public dataset"], "color": "#8b5cf6", "size": 36}}, {"data": {"id": "references/joins/events___ads_clickstats", "label": "Join Google Analytics Events to Google Ads Clicks", "type": "Reference", "description": "Join Google Analytics event data with Google Ads click data.", "resource": "https://developers.google.com/analytics/bigquery/basic-queries", "tags": ["join", "Google Ads"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/avg_pageviews", "label": "Average Pageviews", "type": "Reference", "description": "The average number of pageviews per user.", "resource": "https://developers.google.com/analytics/bigquery/basic-queries", "tags": ["metric"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/avg_spend_per_purchase_session_by_user", "label": "Average Spend Per Purchase Session By User", "type": "Reference", "description": "The average amount of money spent per purchase session for each individual user.", "resource": "https://developers.google.com/analytics/bigquery/advanced-queries", "tags": ["metric", "ecommerce"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/avg_transactions_per_purchaser", "label": "Average Transactions Per Purchaser", "type": "Reference", "description": "The average number of transactions made by purchasers.", "resource": "https://developers.google.com/analytics/bigquery/basic-queries", "tags": ["metric", "ecommerce"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/day_count", "label": "Day Count", "type": "Reference", "description": "Total number of unique days.", "resource": "https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "tags": ["metric"], "color": "#10b981", "size": 30}}, {"data": {"id": "references/metrics/event_count", "label": "Event Count", "type": "Reference", "description": "Total number of events.", "resource": "https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "tags": ["metric"], "color": "#10b981", "size": 30}}, {"data": {"id": "references/metrics/new_user_count", "label": "New User Count", "type": "Reference", "description": "The number of unique users who triggered a first_visit or first_open event.", "resource": "https://developers.google.com/analytics/bigquery/basic-queries", "tags": ["metric"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/overall_avg_spend_per_purchase_session", "label": "Overall Average Spend Per Purchase Session", "type": "Reference", "description": "The overall average amount spent across all unique purchase sessions.", "resource": "https://developers.google.com/analytics/bigquery/advanced-queries", "tags": ["metric", "ecommerce"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/metrics/user_count", "label": "User Count", "type": "Reference", "description": "Total number of unique users.", "resource": "https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "tags": ["metric"], "color": "#10b981", "size": 30}}, {"data": {"id": "tables/events_", "label": "Events table (Google Analytics BigQuery Export)", "type": "BigQuery Table", "description": "Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce/tables/events_*", "tags": ["events", "Google Analytics", "BigQuery", "ecommerce", "schema", "basic queries", "advanced queries"], "color": "#3b82f6", "size": 90}}], "edges": [{"data": {"id": "tables/events___references/metrics/event_count", "source": "tables/events_", "target": "references/metrics/event_count"}}, {"data": {"id": "tables/events___references/metrics/user_count", "source": "tables/events_", "target": "references/metrics/user_count"}}, {"data": {"id": "tables/events___references/metrics/day_count", "source": "tables/events_", "target": "references/metrics/day_count"}}, {"data": {"id": "tables/events___references/metrics/new_user_count", "source": "tables/events_", "target": "references/metrics/new_user_count"}}, {"data": {"id": "tables/events___references/metrics/avg_transactions_per_purchaser", "source": "tables/events_", "target": "references/metrics/avg_transactions_per_purchaser"}}, {"data": {"id": "tables/events___references/metrics/avg_pageviews", "source": "tables/events_", "target": "references/metrics/avg_pageviews"}}, {"data": {"id": "tables/events___references/metrics/avg_spend_per_purchase_session_by_user", "source": "tables/events_", "target": "references/metrics/avg_spend_per_purchase_session_by_user"}}, {"data": {"id": "tables/events___references/metrics/overall_avg_spend_per_purchase_session", "source": "tables/events_", "target": "references/metrics/overall_avg_spend_per_purchase_session"}}, {"data": {"id": "tables/events___references/joins/events___ads_clickstats", "source": "tables/events_", "target": "references/joins/events___ads_clickstats"}}], "bodies": {"datasets/ga4_obfuscated_sample_ecommerce": "# Overview\nThe `ga4_obfuscated_sample_ecommerce` dataset contains obfuscated Google Analytics BigQuery event export data for three months (November 2020 to January 2021) from the Google Merchandise Store. This public dataset is available in BigQuery and emulates a real-world dataset.\n\n# Pre-requisites\nTo work with this dataset, you need access to a Google Cloud project with the BigQuery API enabled. You can use BigQuery Sandbox mode or the Free usage tier for exploration and sample queries.\n\n# Limitations\nThe dataset contains obfuscated data with placeholder values like `<Other>`, `NULL`, and `''`. Due to obfuscation, the internal consistency of the dataset might be somewhat limited. It cannot be compared to the Google Analytics Demo Account.\n\n# Using the dataset\nYou can access the `ga4_obfuscated_sample_ecommerce` dataset via the BigQuery UI in the Cloud Console.\n\n## Sample Query\nThe following query shows the number of unique events, users, and days in the dataset:\n\n```sql\nSELECT\n  COUNT(*) AS event_count,\n  COUNT(DISTINCT user_pseudo_id) AS user_count,\n  COUNT(DISTINCT event_date) AS day_count\nFROM `bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*`\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "references/joins/events___ads_clickstats": "Join Google Analytics event data with Google Ads click data.\n\n```sql\nGA_EVENTS.collected_traffic_source.gclid = ADS_CLICKS.gclid\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/basic-queries", "references/metrics/avg_pageviews": "The average number of pageviews per user.\n\n```sql\nSUM(page_view_count) / COUNT(*)\n-- where page_view_count is COUNTIF(event_name = 'page_view') per user\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/basic-queries", "references/metrics/avg_spend_per_purchase_session_by_user": "The average amount of money spent per purchase session for each individual user.\n\n```sql\nAVG(total_session_spend)\n-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/advanced-queries", "references/metrics/avg_transactions_per_purchaser": "The average number of transactions made by purchasers.\n\n```sql\nCOUNT(*) / COUNT(DISTINCT user_pseudo_id)\n-- for events where event_name IN ('in_app_purchase', 'purchase')\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/basic-queries", "references/metrics/day_count": "Total number of unique days.\n\n```sql\nCOUNT(DISTINCT event_date)\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "references/metrics/event_count": "Total number of events.\n\n```sql\nCOUNT(*)\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "references/metrics/new_user_count": "The number of unique users who triggered a `first_visit` or `first_open` event.\n\n```sql\nSUM(is_new_user)\n-- where is_new_user is MAX(IF(event_name IN ('first_visit', 'first_open'), 1, 0)) grouped by user_pseudo_id\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/basic-queries", "references/metrics/overall_avg_spend_per_purchase_session": "The overall average amount spent across all unique purchase sessions.\n\n```sql\nAVG(total_session_spend)\n-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/advanced-queries", "references/metrics/user_count": "Total number of unique users.\n\n```sql\nCOUNT(DISTINCT user_pseudo_id)\n```\n\n# Citations\n- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset", "tables/events_": "# Overview\nThe `events_` table is a sharded BigQuery table containing Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.\n\n# Metrics\n- [Event Count](/references/metrics/event_count.md) \u2014 Total number of events.\n- [User Count](/references/metrics/user_count.md) \u2014 Total number of unique users.\n- [Day Count](/references/metrics/day_count.md) \u2014 Total number of unique days.\n- [New User Count](/references/metrics/new_user_count.md) \u2014 The number of unique users who triggered a first_visit or first_open event.\n- [Average Transactions Per Purchaser](/references/metrics/avg_transactions_per_purchaser.md) \u2014 The average number of transactions made by purchasers.\n- [Average Pageviews](/references/metrics/avg_pageviews.md) \u2014 The average number of pageviews per user.\n- [Average Spend Per Purchase Session By User](/references/metrics/avg_spend_per_purchase_session_by_user.md) \u2014 The average amount of money spent per purchase session for each individual user.\n- [Overall Average Spend Per Purchase Session](/references/metrics/overall_avg_spend_per_purchase_session.md) \u2014 The overall average amount spent across all unique purchase sessions.\n\n# Schema\nThe `events_YYYYMMDD` table, created daily, and `events_intraday_YYYYMMDD` (for streaming export) contain the following fields:\n\n## event\nThe event fields contain information that uniquely identifies an event.\n\n- `batch_event_index` (INTEGER): A number indicating the sequential order of each event within a batch based on their order of occurrence on the device.\n- `batch_ordering_id` (INTEGER): A monotonically increasing number that is incremented each time a network request is sent from a given page.\n- `batch_page_id` (INTEGER): A sequential number assigned to a page that increases for each subsequent page within an engagement.\n- `event_date` (STRING): The date when the event was logged (YYYYMMDD format in the registered timezone of your app).\n- `event_timestamp` (INTEGER): The time (in microseconds, UTC) when the event was received by Google Analytics. Multiple events can share the same timestamp if sent in the same request.\n- `event_previous_timestamp` (INTEGER): The time (in microseconds, UTC) when the previous event happened.\n- `event_name` (STRING): The name of the event.\n- `event_value_in_usd` (FLOAT): The currency-converted value (in USD) of the event's \"value\" parameter.\n- `event_bundle_sequence_id` (INTEGER): The sequential ID of the bundle in which these events were uploaded.\n- `event_server_timestamp_offset` (INTEGER): Timestamp offset between collection time and upload time in micros.\n\n### event_params RECORD\nThe `event_params` RECORD can store campaign-level and contextual event parameters as well as any user-defined event parameters. The `event_params` RECORD is repeated for each key that is associated with an event.\n\n- `event_params.key` (STRING): The name of the event parameter.\n- `event_params.value` (RECORD): A record containing the event parameter's value.\n    - `event_params.value.string_value` (STRING): If the event parameter is represented by a string, such as a URL or campaign name, it is populated in this field.\n    - `event_params.value.int_value` (INTEGER): If the event parameter is represented by an integer, it is populated in this field.\n    - `event_params.value.double_value` (FLOAT): If the event parameter is represented by a double value, it is populated in this field.\n    - `event_params.value.float_value` (FLOAT): If the event parameter is represented by a floating point value, it is populated in this field. This field is not currently in use.\n- `event_params.key = 'page_location'` (STRING): The URL of the page viewed.\n- `event_params.key = 'page_title'` (STRING): The title of the page viewed.\n- `event_params.key = 'ga_session_id'` (INTEGER): The ID of the Google Analytics session.\n- `event_params.key = 'ga_session_number'` (INTEGER): The sequential number of the session for a user.\n\n### item_params RECORD\nThe `item_params` RECORD can store item parameters as well as any user-defined item parameters.\n\n- `item_params.key` (STRING): The name of the item parameter.\n- `item_params.value` (RECORD): A record containing the item parameter\u2019s value.\n    - `item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.\n    - `item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.\n    - `item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.\n    - `item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.\n\n## user\nThe user fields contain information that uniquely identifies the user associated with the event.\n\n- `is_active_user` (BOOLEAN): Whether the user was active (True) or inactive (False) at any point in the calendar day. This field is only populated in the daily tables (`events_YYYYMMDD`).\n- `user_id` (STRING): The unique ID assigned to a user.\n- `user_pseudo_id` (STRING): The pseudonymous id (e.g., app instance ID) for the user. A unique identifier that is assigned to a user when they first open the app or visit the site.\n- `user_first_touch_timestamp` (INTEGER): The time (in microseconds) at which the user first opened the app or visited the site.\n\n### privacy_info fields\nThe `privacy_info` fields contain information based on the consent status of a user when consent mode is enabled.\n\n- `privacy_info.ads_storage` (STRING): Whether ad targeting is enabled for a user. Possible values: Yes, No, Unset\n- `privacy_info.analytics_storage` (STRING): Whether Analytics storage is enabled for the user. Possible values: Yes, No, Unset\n- `privacy_info.uses_transient_token` (STRING): Whether a web user has denied Analytics storage and the developer has enabled measurement without cookies based on transient tokens in server data. Possible values: Yes, No, Unset\n\n### user_properties RECORD\nThe `user_properties` RECORD contains any user properties that you have set. It is repeated for each key that is associated with a user.\n\n- `user_properties.key` (STRING): The name of the user property.\n- `user_properties.value` (RECORD): A record for the user property value.\n    - `user_properties.value.string_value` (STRING): The string value of the user property.\n    - `user_properties.value.int_value` (INTEGER): The integer value of the user property.\n    - `user_properties.value.double_value` (FLOAT): The double value of the user property.\n    - `user_properties.value.float_value` (FLOAT): This field is currently unused.\n    - `user_properties.value.set_timestamp_micros` (INTEGER): The time (in microseconds) at which the user property was last set.\n\n### user_ltv RECORD\nThe `user_ltv` RECORD contains Lifetime Value information about the user. This RECORD is not populated in intraday tables.\n\n- `user_ltv.revenue` (FLOAT): The Lifetime Value (revenue) of the user. This field is not populated in intraday tables.\n- `user_ltv.currency` (STRING): The Lifetime Value (currency) of the user. This field is not populated in intraday tables.\n\n## device\nThe device RECORD contains information about the device from which the event originated.\n\n- `device.category` (STRING): The device category (mobile, tablet, desktop).\n- `device.mobile_brand_name` (STRING): The device brand name.\n- `device.mobile_model_name` (STRING): The device model name.\n- `device.mobile_marketing_name` (STRING): The device marketing name.\n- `device.mobile_os_hardware_model` (STRING): The device model information retrieved directly from the operating system.\n- `device.operating_system` (STRING): The operating system of the device.\n- `device.operating_system_version` (STRING): The OS version.\n- `device.vendor_id` (STRING): IDFV (present only if IDFA is not collected).\n- `device.advertising_id` (STRING): Advertising ID/IDFA.\n- `device.language` (STRING): The OS language.\n- `device.time_zone_offset_seconds` (INTEGER): The offset from GMT in seconds.\n- `device.is_limited_ad_tracking` (BOOLEAN): The device's Limit Ad Tracking setting. On iOS14+, returns false if the IDFA is non-zero.\n- `device.web_info.browser` (STRING): The browser in which the user viewed content.\n- `device.web_info.browser_version` (STRING): The version of the browser in which the user viewed content.\n- `device.web_info.hostname` (STRING): The hostname associated with the logged event.\n\n## geo\nThe geo RECORD contains information about the geographic location where the event was initiated.\n\n- `geo.continent` (STRING): The continent from which events were reported, based on IP address.\n- `geo.sub_continent` (STRING): The subcontinent from which events were reported, based on IP address.\n- `geo.country` (STRING): The country from which events were reported, based on IP address.\n- `geo.region` (STRING): The region from which events were reported, based on IP address.\n- `geo.metro` (STRING): The metro from which events were reported, based on IP address.\n- `geo.city` (STRING): The city from which events were reported, based on IP address.\n\n## app_info\nThe `app_info` RECORD contains information about the app in which the event was initiated.\n\n- `app_info.id` (STRING): The package name or bundle ID of the app.\n- `app_info.firebase_app_id` (STRING): The Firebase App ID associated with the app.\n- `app_info.install_source` (STRING): The store that installed the app.\n- `app_info.version` (STRING): The app's versionName (Android) or short bundle version.\n\n## collected_traffic_source\nThe `collected_traffic_source` RECORD contains the traffic source data that was present within the events that were collected.\n\n- `manual_campaign_id` (STRING): The manual campaign id (utm_id) that was collected with the event.\n- `manual_campaign_name` (STRING): The manual campaign name (utm_campaign) that was collected with the event.\n- `manual_source` (STRING): The manual campaign source (utm_source) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.\n- `manual_medium` (STRING): The manual campaign medium (utm_medium) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.\n- `manual_term` (STRING): The manual campaign keyword/term (utm_term) that was collected with the event.\n- `manual_content` (STRING): The additional manual campaign metadata (utm_content) that was collected with the event.\n- `manual_creative_format` (STRING): The manual campaign creative format (utm_creative_format) that was collected with the event.\n- `manual_marketing_tactic` (STRING): The manual campaign marketing tactic (utm_marketing_tactic) that was collected with the event.\n- `manual_source_platform` (STRING): The manual campaign source platform (utm_source_platform) that was collected with the event.\n- `gclid` (STRING): The Google click identifier that was collected with the event.\n- `dclid` (STRING): The DoubleClick Click Identifier for Display and Video 360 and Campaign Manager 360 that was collected with the event.\n- `srsltid` (STRING): The Google Merchant Center identifier that was collected with the event.\n\n## session_traffic_source_last_click\nThe `session_traffic_source_last_click` RECORD contains the last-click attributed session traffic source data across Google ads and manual contexts, where available.\n\n- `session_traffic_source_last_click.manual_campaign.campaign_id` (STRING): The ID of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.campaign_name` (STRING): The name of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.medium` (STRING): The medium of the last clicked manual campaign (e.g., paid search, organic search, email).\n- `session_traffic_source_last_click.manual_campaign.term` (STRING): The keyword/search term of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.content` (STRING): Additional metadata of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.source_platform` (STRING): The platform of the last clicked manual campaign (e.g., search engine, social media).\n- `session_traffic_source_last_click.manual_campaign.source` (STRING): The specific source within the platform of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.creative_format` (STRING): The format of the creative of the last clicked manual campaign.\n- `session_traffic_source_last_click.manual_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked manual campaign.\n- `session_traffic_source_last_click.google_ads_campaign.customer_id` (STRING): The customer ID associated with the Google Ads account.\n- `session_traffic_source_last_click.google_ads_campaign.account_name` (STRING): The name of the Google Ads account.\n- `session_traffic_source_last_click.google_ads_campaign.campaign_id` (STRING): The ID of the Google Ads campaign.\n- `session_traffic_source_last_click.google_ads_campaign.campaign_name` (STRING): The name of the Google Ads campaign.\n- `session_traffic_source_last_click.google_ads_campaign.ad_group_id` (STRING): The ID of the ad group within the Google Ads campaign.\n- `session_traffic_source_last_click.google_ads_campaign.ad_group_name` (STRING): The name of the ad group within the Google Ads campaign.\n- `session_traffic_source_last_click.cross_channel_campaign.campaign_name` (STRING): The name of the last clicked cross-channel campaign.\n- `session_traffic_source_last_click.cross_channel_campaign.campaign_id` (STRING): The ID of the last clicked cross-channel campaign.\n- `session_traffic_source_last_click.cross_channel_campaign.source_platform` (STRING): The platform of the last clicked cross-channel campaign.\n- `session_traffic_source_last_click.cross_channel_campaign.source` (STRING): The specific source within the platform of the last clicked cross-channel campaign.\n- `session_traffic_source_last_click.cross_channel_campaign.medium` (STRING): The medium of the last clicked cross-channel campaign.\n- `session_traffic_source_last_click.sa360_campaign.campaign_name` (STRING): The name of the last clicked SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.source` (STRING): The specific source within the platform of the last clicked SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.medium` (STRING): The medium of the last clicked SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.ad_group_id` (STRING): The ID of the ad group within the SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.ad_group_name` (STRING): The name of the ad group within the SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.campaign_id` (STRING): The ID of the last clicked SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.creative_format` (STRING): The format of the creative of the last clicked SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.engine_account_name` (STRING): The name of the SA360 engine account.\n- `session_traffic_source_last_click.sa360_campaign.engine_account_type` (STRING): The type of engine account containing the SA360 campaign.\n- `session_traffic_source_last_click.sa360_campaign.manager_account_name` (STRING): The name of the SA360 manager account.\n- `session_traffic_source_last_click.dv360_campaign.advertiser_id` (STRING): The ID of the DV360 advertiser.\n- `session_traffic_source_last_click.dv360_campaign.advertiser_name` (STRING): The name of the DV360 advertiser.\n- `session_traffic_source_last_click.dv360_campaign.campaign_id` (STRING): The ID of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.campaign_name` (STRING): The name of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.creative_id` (STRING): The ID of the creative of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.creative_format` (STRING): The format of the creative of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.creative_name` (STRING): The name of the creative of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.exchange_id` (STRING): The exchange ID of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.exchange_name` (STRING): The exchange name of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.insertion_order_id` (STRING): The ID of the insertion order in the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.insertion_order_name` (STRING): The name of the insertion order in the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.line_item_id` (STRING): The ID of the line item in the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.line_item_name` (STRING): The name of the line item in the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.partner_id` (STRING): The ID of the DV360 partner.\n- `session_traffic_source_last_click.dv360_campaign.partner_name` (STRING): The name of the DV360 partner.\n- `session_traffic_source_last_click.dv360_campaign.source` (STRING): The specific source within the platform of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.dv360_campaign.medium` (STRING): The medium of the last clicked DV360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.account_id` (STRING): The ID of the CM360 account.\n- `session_traffic_source_last_click.cm360_campaign.account_name` (STRING): The name of the CM360 account.\n- `session_traffic_source_last_click.cm360_campaign.advertiser_id` (STRING): The ID of the CM360 advertiser.\n- `session_traffic_source_last_click.cm360_campaign.advertiser_name` (STRING): The name of the CM360 advertiser.\n- `session_traffic_source_last_click.cm360_campaign.campaign_id` (STRING): The ID of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.campaign_name` (STRING): The name of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_id` (STRING): The ID of the creative of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_format` (STRING): The format of the creative of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_name` (STRING): The name of the creative of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_type` (STRING): The creative type of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_type_id` (STRING): The creative type ID of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.creative_version` (STRING): The creative version of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.placement_id` (STRING): The ID of the placement of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.placement_cost_structure` (STRING): The placement cost structure of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.placement_name` (STRING): The name of the placement of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.rendering_id` (STRING): The rendering ID of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.site_id` (STRING): The site ID of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.site_name` (STRING): The site name of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.source` (STRING): The specific source of the last clicked CM360 campaign.\n- `session_traffic_source_last_click.cm360_campaign.medium` (STRING): The medium of the last clicked CM360 campaign.\n\n## traffic_source\nThe `traffic_source` RECORD contains information about the traffic source that first acquired the user. This record is not populated in intraday tables.\n\n- `traffic_source.name` (STRING): Name of the marketing campaign that first acquired the user. This field is not populated in intraday tables.\n- `traffic_source.medium` (STRING): Name of the medium (paid search, organic search, email, etc.) that first acquired the user. This field is not populated in intraday tables.\n- `traffic_source.source` (STRING): Name of the network that first acquired the user. This field is not populated in intraday tables.\n\n## stream and platform\nThe stream and platform fields contain information about the stream and the app platform.\n\n- `stream_id` (STRING): The numeric ID of the data stream from which the event originated.\n- `platform` (STRING): The data stream platform (Web, IOS or Android) from which the event originated.\n\n## ecommerce\nThis ecommerce RECORD contains information about any ecommerce events that have been setup on a website or app.\n\n- `ecommerce.total_item_quantity` (INTEGER): Total number of items in this event, which is the sum of items.quantity.\n- `ecommerce.purchase_revenue_in_usd` (FLOAT): Purchase revenue of this event, represented in USD with standard unit. Populated for purchase event only.\n- `ecommerce.purchase_revenue` (FLOAT): Purchase revenue of this event, represented in local currency with standard unit. Populated for purchase event only.\n- `ecommerce.refund_value_in_usd` (FLOAT): The amount of refund in this event, represented in USD with standard unit. Populated for refund event only.\n- `ecommerce.refund_value` (FLOAT): The amount of refund in this event, represented in local currency with standard unit. Populated for refund event only.\n- `ecommerce.shipping_value_in_usd` (FLOAT): The shipping cost in this event, represented in USD with standard unit.\n- `ecommerce.shipping_value` (FLOAT): The shipping cost in this event, represented in local currency.\n- `ecommerce.tax_value_in_usd` (FLOAT): The tax value in this event, represented in USD with standard unit.\n- `ecommerce.tax_value` (FLOAT): The tax value in this event, represented in local currency with standard unit.\n- `ecommerce.transaction_id` (STRING): The transaction ID of the ecommerce transaction.\n- `ecommerce.unique_items` (INTEGER): The number of unique items in this event, based on item_id, item_name, and item_brand.\n\n## items\nThe `items` RECORD contains information about items included in an event. It is repeated for each item.\n\n- `items.item_id` (STRING): The ID of the item.\n- `items.item_name` (STRING): The name of the item.\n- `items.item_brand` (STRING): The brand of the item.\n- `items.item_variant` (STRING): The variant of the item.\n- `items.item_category` (STRING): The category of the item.\n- `items.item_category2` (STRING): The sub category of the item.\n- `items.item_category3` (STRING): The sub category of the item.\n- `items.item_category4` (STRING): The sub category of the item.\n- `items.item_category5` (STRING): The sub category of the item.\n- `items.price_in_usd` (FLOAT): The price of the item, in USD with standard unit.\n- `items.price` (FLOAT): The price of the item in local currency.\n- `items.quantity` (INTEGER): The quantity of the item. Quantity set to 1 if not specified.\n- `items.item_revenue_in_usd` (FLOAT): The revenue of this item, calculated as price_in_usd * quantity. It is populated for purchase events only, in USD with standard unit.\n- `items.item_revenue` (FLOAT): The revenue of this item, calculated as price * quantity. It is populated for purchase events only, in local currency with standard unit.\n- `items.item_refund_in_usd` (FLOAT): The refund value of this item, calculated as price_in_usd * quantity. It is populated for refund events only, in USD with standard unit.\n- `items.item_refund` (FLOAT): The refund value of this item, calculated as price * quantity. It is populated for refund events only, in local currency with standard unit.\n- `items.coupon` (STRING): Coupon code applied to this item.\n- `items.affiliation` (STRING): A product affiliation to designate a supplying company or brick and mortar store location.\n- `items.location_id` (STRING): The location associated with the item.\n- `items.item_list_id` (STRING): The ID of the list in which the item was presented to the user.\n- `items.item_list_name` (STRING): The name of the list in which the item was presented to the user.\n- `items.item_list_index` (STRING): The position of the item in a list.\n- `items.promotion_id` (STRING): The ID of a product promotion.\n- `items.promotion_name` (STRING): The name of a product promotion.\n- `items.creative_name` (STRING): The name of a creative used in a promotional spot.\n- `items.creative_slot` (STRING): The name of a creative slot.\n\n### item_params RECORD (nested under items)\nThe `item_params` RECORD stores the custom item parameters that you defined.\n\n- `items.item_params.key` (STRING): The name of the item parameter.\n- `items.item_params.value` (RECORD): A record containing the item parameter\u2019s value.\n    - `items.item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.\n    - `items.item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.\n    - `items.item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.\n    - `items.item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.\n\n## publisher (Early access only)\nThe publisher RECORD contains information about events sourced from a publisher integration related to the display of ads, that is, AdMob.\n\n- `publisher` (RECORD): A record of publisher data coming from AdMob. This field is not populated in intraday tables and Fresh Daily BigQuery export.\n- `publisher.ad_revenue_in_usd` (FLOAT): Estimated ad revenue resulting from this event, represented in USD. Populated for ad impression events only. This field is not populated in intraday tables and Fresh Daily BigQuery export.\n- `publisher.ad_format` (STRING): Describes the way ads appeared and where they were located. Typical formats include \u2018Interstitial\u2019, \u2018Banner\u2019, \u2018Rewarded\u2019, and \u2018Native advanced\u2019. This field is not populated in intraday tables and Fresh Daily BigQuery export.\n- `publisher.ad_source_name` (STRING): The source network that served an ad. Typical sources include, \u2018AdMob Network\u2019, \u2018Meta audience Network\u2019, and \u2018Mediated house ads\u2019. This field is not populated in intraday tables and Fresh Daily BigQuery export.\n- `publisher.ad_unit_id` (STRING): The name you chose to describe this Ad unit. Ad units are containers that you place in your apps to show ads to users. This field is not populated in intraday tables and Fresh Daily BigQuery export.\n\n# Joins\n- [Google Analytics Events to Google Ads Clicks](/references/joins/events___ads_clickstats.md) \u2014 join on `collected_traffic_source.gclid` to attach Google Ads data to events.\n\n# Citations\n- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset\n- https://support.google.com/analytics/answer/7029846\n- https://developers.google.com/analytics/bigquery/basic-queries\n- https://developers.google.com/analytics/bigquery/advanced-queries"}, "types": ["BigQuery Dataset", "BigQuery Table", "Reference"], "palette": {"BigQuery Dataset": "#8b5cf6", "BigQuery Table": "#3b82f6", "Reference": "#10b981"}};
</script>
<script>
(function () {
  const bundle = window.BUNDLE;
  const bundleName = window.BUNDLE_NAME;
  document.title = `${bundleName} — OKF Viewer`;
  document.getElementById("bundle-name").textContent = bundleName;

  // Populate type filter
  const typeSelect = document.getElementById("filter-type");
  for (const t of bundle.types) {
    const opt = document.createElement("option");
    opt.value = t;
    opt.textContent = t;
    typeSelect.appendChild(opt);
  }

  // Build reverse-link index for backlinks
  const backlinks = {};
  for (const edge of bundle.edges) {
    const { source, target } = edge.data;
    (backlinks[target] ||= []).push(source);
  }

  // Look up node label/type by id
  const nodeIndex = {};
  for (const n of bundle.nodes) nodeIndex[n.data.id] = n.data;

  const cy = cytoscape({
    container: document.getElementById("graph"),
    elements: [...bundle.nodes, ...bundle.edges],
    style: [
      {
        selector: "node",
        style: {
          "background-color": "data(color)",
          "label": "data(label)",
          "color": "#0f172a",
          "font-size": 11,
          "text-valign": "bottom",
          "text-margin-y": 4,
          "text-wrap": "wrap",
          "text-max-width": 120,
          "width": "data(size)",
          "height": "data(size)",
          "border-width": 1,
          "border-color": "#0f172a",
        },
      },
      {
        selector: "node:selected",
        style: {
          "border-width": 3,
          "border-color": "#f59e0b",
        },
      },
      {
        selector: "edge",
        style: {
          "width": 1.5,
          "line-color": "#cbd5e1",
          "target-arrow-color": "#cbd5e1",
          "target-arrow-shape": "triangle",
          "curve-style": "bezier",
          "arrow-scale": 0.9,
        },
      },
      {
        selector: "edge:selected",
        style: {
          "line-color": "#f59e0b",
          "target-arrow-color": "#f59e0b",
          "width": 2.5,
        },
      },
      {
        selector: ".dim",
        style: { "opacity": 0.15 },
      },
    ],
    layout: { name: "cose", animate: false, padding: 30 },
    wheelSensitivity: 0.2,
  });

  cy.on("tap", "node", (evt) => showDetail(evt.target.id()));
  cy.on("tap", (evt) => {
    if (evt.target === cy) clearSelection();
  });

  document.getElementById("layout").addEventListener("change", (e) => {
    cy.layout({ name: e.target.value, animate: false, padding: 30 }).run();
  });

  document.getElementById("reset").addEventListener("click", () => {
    cy.fit(null, 30);
    clearSelection();
  });

  document.getElementById("search").addEventListener("input", (e) => {
    const q = e.target.value.trim().toLowerCase();
    if (!q) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      const d = n.data();
      const hay =
        (d.label || "").toLowerCase() + " " +
        d.id.toLowerCase() + " " +
        (d.tags || []).join(" ").toLowerCase();
      n.toggleClass("dim", !hay.includes(q));
    });
    cy.edges().forEach((edge) => {
      const src = edge.source();
      const tgt = edge.target();
      edge.toggleClass("dim", src.hasClass("dim") || tgt.hasClass("dim"));
    });
  });

  document.getElementById("filter-type").addEventListener("change", (e) => {
    const t = e.target.value;
    if (!t) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      n.toggleClass("dim", n.data("type") !== t);
    });
    cy.edges().forEach((edge) => {
      edge.toggleClass("dim", edge.source().hasClass("dim") || edge.target().hasClass("dim"));
    });
  });

  function clearSelection() {
    cy.elements().unselect();
    document.getElementById("detail-empty").hidden = false;
    document.getElementById("detail-content").hidden = true;
  }

  function showDetail(conceptId) {
    const data = nodeIndex[conceptId];
    if (!data) return;
    cy.elements().unselect();
    const node = cy.getElementById(conceptId);
    if (node) node.select();

    document.getElementById("detail-empty").hidden = true;
    const content = document.getElementById("detail-content");
    content.hidden = false;

    const chip = document.getElementById("detail-type");
    chip.textContent = data.type;
    chip.style.background = data.color;

    document.getElementById("detail-title").textContent = data.label;
    document.getElementById("detail-id").textContent = conceptId;
    document.getElementById("detail-description").textContent = data.description || "—";

    const resourceEl = document.getElementById("detail-resource");
    resourceEl.innerHTML = "";
    if (data.resource) {
      const a = document.createElement("a");
      a.href = data.resource;
      a.textContent = data.resource;
      a.target = "_blank";
      a.rel = "noopener";
      a.className = "external";
      resourceEl.appendChild(a);
    } else {
      resourceEl.textContent = "—";
    }

    const tagsEl = document.getElementById("detail-tags");
    tagsEl.innerHTML = "";
    if (data.tags && data.tags.length) {
      for (const t of data.tags) {
        const span = document.createElement("span");
        span.className = "tag";
        span.textContent = t;
        tagsEl.appendChild(span);
      }
    } else {
      tagsEl.textContent = "—";
    }

    const body = bundle.bodies[conceptId] || "";
    const html = marked.parse(body, { breaks: false, gfm: true });
    const bodyEl = document.getElementById("detail-body");
    bodyEl.innerHTML = html;
    rewriteInternalLinks(bodyEl);

    const bl = backlinks[conceptId] || [];
    const blSection = document.getElementById("detail-backlinks");
    const blList = document.getElementById("backlinks-list");
    blList.innerHTML = "";
    if (bl.length) {
      blSection.hidden = false;
      for (const src of bl) {
        const li = document.createElement("li");
        const a = document.createElement("a");
        a.textContent = nodeIndex[src]?.label || src;
        a.dataset.target = src;
        a.addEventListener("click", () => showDetail(src));
        li.appendChild(a);
        const muted = document.createElement("span");
        muted.className = "muted";
        muted.textContent = ` (${src})`;
        li.appendChild(muted);
        blList.appendChild(li);
      }
    } else {
      blSection.hidden = true;
    }

    cy.animate({ center: { eles: node }, zoom: Math.max(cy.zoom(), 1.0) }, { duration: 200 });
  }

  function rewriteInternalLinks(root) {
    root.querySelectorAll("a[href]").forEach((a) => {
      const href = a.getAttribute("href");
      if (!href) return;
      if (href.startsWith("/") && href.endsWith(".md")) {
        const target = href.slice(1, -3);
        if (nodeIndex[target]) {
          a.className = "internal";
          a.setAttribute("href", "javascript:void(0)");
          a.addEventListener("click", (e) => {
            e.preventDefault();
            showDetail(target);
          });
          return;
        }
      }
      a.className = "external";
      a.setAttribute("target", "_blank");
      a.setAttribute("rel", "noopener");
    });
  }

  // Auto-show the first node (a dataset if available, else first concept)
  const initial =
    bundle.nodes.find((n) => n.data.type === "BigQuery Dataset") ||
    bundle.nodes[0];
  if (initial) showDetail(initial.data.id);
})();

</script>
</body>
</html>


================================================
FILE: okf\bundles\stackoverflow\datasets\index.md
================================================
# BigQuery Dataset

* [Stack Overflow Public Dataset](stackoverflow.md) - The Stack Overflow public dataset contains a variety of tables related to Stack Overflow user activity, posts, and tags. This dataset is no longer actively updated.


================================================
FILE: okf\bundles\stackoverflow\datasets\stackoverflow.md
================================================
---
type: BigQuery Dataset
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow
title: Stack Overflow Public Dataset
description: The Stack Overflow public dataset contains a variety of tables related
  to Stack Overflow user activity, posts, and tags. This dataset is no longer actively
  updated.
tags: Stack Overflow, public data, community, Q&A
timestamp: '2026-05-28T23:25:15+00:00'
---

The Stack Overflow public dataset in BigQuery provides a comprehensive collection of data from the popular question and answer platform. This dataset is a valuable resource for researchers and developers interested in analyzing trends in programming questions, user interactions, and the evolution of technology discussions. It encompasses various aspects of the platform, including user profiles, questions, answers, comments, votes, badges awarded, and associated tags.

This dataset was last updated on 2022-11-25, and it is important to note that it is no longer actively maintained. Despite this, it remains a rich historical archive of Stack Overflow activity. The dataset is hosted in the `US` multi-region.

# Schema

As a BigQuery Dataset, `stackoverflow` serves as a container for numerous tables rather than having a schema itself. The tables within this dataset collectively represent the structure of Stack Overflow data. Key tables include:

*   [badges](../tables/badges.md): Information about badges awarded to users.
*   [comments](../tables/comments.md): User comments on posts.
*   [post_history](../tables/post_history.md): Historical revisions and events for posts.
*   [posts_answers](../tables/posts_answers.md): Detailed information about answers.
*   [posts_questions](../tables/posts_questions.md): Detailed information about questions.
*   [tags](../tables/tags.md): Information about tags used to categorize posts.
*   [users](../tables/users.md): User profiles and statistics.
*   [votes](../tables/votes.md): Records of upvotes and downvotes on posts and comments.

# Common query patterns

List all tables within the `stackoverflow` dataset:

```sql
SELECT table_id
FROM `bigquery-public-data.stackoverflow.INFORMATION_SCHEMA.TABLES`
WHERE table_schema = 'stackoverflow';
```

Example of querying the `posts_questions` table:

```sql
SELECT
  id,
  title,
  view_count,
  creation_date
FROM
  `bigquery-public-data.stackoverflow.posts_questions`
WHERE
  creation_date BETWEEN '2021-01-01' AND '2021-01-31'
ORDER BY
  view_count DESC
LIMIT 5;
```

# Citations

[1] [Stack Overflow Public Dataset](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow)
[2] [Stack Overflow Official Website](https://stackoverflow.com/)


================================================
FILE: okf\bundles\stackoverflow\index.md
================================================
# Subdirectories

* [datasets](datasets/index.md) - The Stack Overflow public dataset contains a variety of tables related to Stack Overflow user activity, posts, and tags. This dataset is no longer actively updated.
* [references](references/index.md) - This directory contains enumerated types, data structures, and internal references for Stack Exchange content, moderation, and the Data Explorer.
* [tables](tables/index.md) - The directory contains structured data tables describing Stack Overflow content, users, and activity.


================================================
FILE: okf\bundles\stackoverflow\references\badge_classes.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Badge Classes
description: Enumerated classes for badges awarded on Stack Exchange sites.
tags:
- badges
- classes
- enum
- stackoverflow
timestamp: '2026-05-28T23:32:40+00:00'
---

# Badge Classes

This document defines the enumerated classes for badges awarded on Stack Exchange sites.

- `1`: Gold
- `2`: Silver
- `3`: Bronze

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\close_as_off_topic_reason_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Close As Off-Topic Reason Types
description: Defines the types and guidance for reasons why a post might be closed
  as off-topic.
tags:
- close reasons
- off-topic
- enum
- schema
- data dump
timestamp: '2026-05-28T23:32:53+00:00'
---

This table defines the enumerated reasons and associated guidance for why a post might be closed as off-topic.

## Schema
- `Id` (INTEGER) - Unique identifier for the reason type.
- `IsUniversal` (BOOLEAN) - Indicates if the reason is universally applicable.
- `InputTitle` (STRING) - The title shown for input.
- `MarkdownInputGuidance` (STRING) - Markdown guidance shown while flagging/voting.
- `MarkdownPostOwnerGuidance` (STRING) - Markdown guidance shown to the original poster when closed.
- `MarkdownPublicGuidance` (STRING) - Markdown guidance shown to privileged users when closed.
- `MarkdownConcensusDescription` (STRING) - Markdown description shown above the public or post owner guidance. Nullable.
- `CreationDate` (TIMESTAMP) - The date and time the reason type was created.
- `CreationModeratorId` (INTEGER) - The ID of the moderator who created the reason type. Links to the `users` table.
- `ApprovalDate` (TIMESTAMP) - The date and time the reason type was approved.
- `ApprovalModeratorId` (INTEGER) - The ID of the moderator who approved the reason type.
- `DeactivationDate` (TIMESTAMP) - The date and time the reason type was deactivated.
- `DeactivationModeratorId` (INTEGER) - The ID of the moderator who deactivated the reason type. Links to the `users` table.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\close_reason_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Close Reason Types
description: Enumerated types for reasons why a post might be closed on Stack Exchange.
tags:
- close reasons
- moderation
- enum
- schema
- data dump
timestamp: '2026-05-28T23:33:07+00:00'
---

This document defines the enumerated types for reasons why a post might be closed on Stack Exchange sites, as found in the `PendingFlags` and `PostHistory` tables.

## Old Close Reasons
- `1`: Exact Duplicate
- `2`: Off-topic
- `3`: Subjective and argumentative
- `4`: Not a real question
- `7`: Too localized
- `10`: General reference
- `20`: Noise or pointless (Meta sites only)

## Current Close Reasons
- `101`: Duplicate
- `102`: Off-topic
- `103`: Unclear what you\'re asking
- `104`: Too broad
- `105`: Primarily opinion-based

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\creative_commons_by_sa_4_0_license.md
================================================
---
type: Reference
resource: https://archive.org/details/stackexchange
title: Creative Commons Attribution-ShareAlike 4.0 International License
description: Details the attribution requirements for content licensed under CC BY-SA
  4.0 for the Stack Exchange network.
tags:
- licensing
- creative commons
- attribution
- data dump
timestamp: '2026-05-28T23:36:58+00:00'
---

All user content contributed to the Stack Exchange network is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0). While this license is permissive, it requires attribution.

## Attribution Requirements for Stack Exchange Content:
1.  Visually display or otherwise indicate the source of the content as coming from the Stack Exchange Network. This requirement is satisfied with a discreet text blurb, or some other unobtrusive but clear visual indication.
2.  Ensure that any Internet use of the content includes a hyperlink directly to the original question on the source site on the Network (e.g., `http://stackoverflow.com/questions/12345`).
3.  Visually display or otherwise clearly indicate the author names for every question and answer used.
4.  Ensure that any Internet use of the content includes a hyperlink for each author name directly back to his or her user profile page on the source site on the Network (e.g., `http://stackoverflow.com/users/12345/username`), directly to the Stack Exchange domain, in standard HTML (i.e. not through a Tinyurl or other such indirect hyperlink, form of obfuscation or redirection), without any “nofollow” command or any other such means of avoiding detection by search engines, and visible even with JavaScript disabled.

For more information, see the [Stack Exchange Terms of Service](https://stackexchange.com/legal/terms-of-service).

# Citations
- [Stack Exchange Data Dump : Internet Archive](https://archive.org/details/stackexchange)
- [Creative Commons Attribution-ShareAlike 4.0 International License](https://creativecommons.org/licenses/by-sa/4.0/)
- [Stack Exchange Terms of Service](https://stackexchange.com/legal/terms-of-service)


================================================
FILE: okf\bundles\stackoverflow\references\flag_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Flag Types
description: Enumerated types for flags related to post moderation on Stack Exchange.
tags:
- flags
- moderation
- enum
- schema
- data dump
timestamp: '2026-05-28T23:32:58+00:00'
---

This document defines the enumerated types for flags related to post moderation on Stack Exchange sites, as found in the `PendingFlags` table.

- `13`: Canned flag for closure
- `14`: Vote to close
- `15`: Vote to reopen

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\index.md
================================================
# Reference

* [Badge Classes](badge_classes.md) - Enumerated classes for badges awarded on Stack Exchange sites.
* [Close As Off-Topic Reason Types](close_as_off_topic_reason_types.md) - Defines the types and guidance for reasons why a post might be closed as off-topic.
* [Close Reason Types](close_reason_types.md) - Enumerated types for reasons why a post might be closed on Stack Exchange.
* [Creative Commons Attribution-ShareAlike 4.0 International License](creative_commons_by_sa_4_0_license.md) - Details the attribution requirements for content licensed under CC BY-SA 4.0 for the Stack Exchange network.
* [Flag Types](flag_types.md) - Enumerated types for flags related to post moderation on Stack Exchange.
* [Link Types](link_types.md) - Enumerated types for links between posts on Stack Exchange.
* [Pending Flags](pending_flags.md) - Stores information about pending flags and votes related to post moderation.
* [Post Feedback](post_feedback.md) - Collects up and down votes from anonymous visitors and unregistered users.
* [Post History Type IDs](post_history_type_ids.md) - Enumerated types for post history events on Stack Exchange.
* [Post Notice Class IDs](post_notice_class_ids.md) - Enumerated class identifiers for different types of post notices.
* [Post Notice Duration IDs](post_notice_duration_ids.md) - Enumerated duration identifiers for post notices.
* [Post Notice Type Definitions](post_notice_type_definitions.md) - Defines the structure and properties of different types of post notices.
* [Post Notice Types](post_notice_types.md) - Enumerated types for notices that can be applied to posts on Stack Exchange.
* [Post Notices](post_notices.md) - Stores information about notices applied to posts, including their type, dates, and associated users.
* [Post Type IDs](post_type_ids.md) - Enumerated IDs for different types of posts on Stack Exchange.
* [Related Questions](related_questions.md) - Provides a mapping between questions and other related questions based on various factors.
* [Review Rejection Reasons](review_rejection_reasons.md) - Defines canned rejection reasons for suggested edits.
* [Review Task Result Types](review_task_result_types.md) - Enumerated types for the outcomes of review tasks on Stack Exchange.
* [Review Task Results](review_task_results.md) - Stores the outcomes and details of completed review tasks.
* [Review Task States](review_task_states.md) - Enumerated states for review tasks on Stack Exchange.
* [Review Task Types](review_task_types.md) - Enumerated types for various review tasks on Stack Exchange.
* [Review Tasks](review_tasks.md) - Stores information about moderation review tasks on Stack Exchange, such as suggested edits or close votes.
* [SEDE Databases](sede_databases.md) - Details the structure of the internal SEDE database metadata table.
* [SEDE Databases Markdown](sede_databases_markdown.md) - Describes a SEDE internal table containing markdown content for database information.
* [SEDE In Each DB Stored Procedure](sede_ineachdb.md) - Describes the SEDE internal stored procedure for running SQL commands across multiple databases.
* [SEDE Sites](sede_sites.md) - Details the structure of the internal SEDE sites metadata table.
* [SEDE Tables](sede_tables.md) - Details the structure of the internal SEDE tables metadata table.
* [SEDE Users](sede_users.md) - Details the structure of the internal SEDE users metadata table.
* [Suggested Edit Votes](suggested_edit_votes.md) - Records votes cast on suggested edits, indicating approval or rejection.
* [Suggested Edits](suggested_edits.md) - Stores information about suggested edits to posts that are awaiting review.
* [Tag Synonyms](tag_synonyms.md) - Stores relationships between synonymous tags on Stack Exchange.
* [Vote Types](vote_types.md) - Enumerated types for votes on posts and other entities across Stack Exchange sites.


================================================
FILE: okf\bundles\stackoverflow\references\link_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Link Types
description: Enumerated types for links between posts on Stack Exchange.
tags:
- links
- enum
- schema
- data dump
timestamp: '2026-05-28T23:34:01+00:00'
---

This document defines the enumerated types for links between posts, as found in the `PostLinks` table.

- `1`: Linked (`PostId` contains a link to `RelatedPostId`)
- `3`: Duplicate (`PostId` is a duplicate of `RelatedPostId`)

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\pending_flags.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Pending Flags
description: Stores information about pending flags and votes related to post moderation.
tags:
- flags
- moderation
- schema
- data dump
timestamp: '2026-05-28T23:33:15+00:00'
---

This table contains information about pending flags and votes related to post moderation, despite its name focusing on "PendingFlags". It includes close-related flags and votes.

## Schema
- `Id` (INTEGER) - Unique identifier for the pending flag.
- `FlagTypeId` (INTEGER) - The type of flag. See the [Flag Types reference](flag_types.md) for possible values:
    - `13`: canned flag for closure
    - `14`: vote to close
    - `15`: vote to reopen
- `PostId` (INTEGER) - The ID of the post the flag is associated with. Links to the `posts` tables (`posts_questions` or `posts_answers`).
- `CreationDate` (TIMESTAMP) - The date and time the flag was created.
- `CloseReasonTypeId` (INTEGER) - The type of close reason, if applicable. See the [Close Reason Types reference](close_reason_types.md) for possible values.
- `CloseAsOffTopicReasonTypeId` (INTEGER) - The specific off-topic reason ID, if `CloseReasonTypeId = 102` (off-topic). Links to the [Close As Off-Topic Reason Types reference](close_as_off_topic_reason_types.md).
- `DuplicateOfQuestionId` (INTEGER) - The ID of the duplicate question, if `CloseReasonTypeId` is `1` or `101` (old or current duplicate). Links to the `posts_questions` table.
- `BelongsOnBaseHostAddress` (STRING) - Indicates the base host address for votes to close and migrate.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_feedback.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Feedback
description: Collects up and down votes from anonymous visitors and unregistered users.
tags:
- feedback
- votes
- anonymous
- schema
- data dump
timestamp: '2026-05-28T23:33:30+00:00'
---

This table collects up and down votes from anonymous visitors and/or unregistered users.

## Schema
- `Id` (INTEGER) - Unique identifier for the post feedback entry.
- `PostId` (INTEGER) - The ID of the post (`posts_questions` or `posts_answers`) the feedback is for.
- `IsAnonymous` (BOOLEAN) - Indicates if the feedback was given anonymously.
- `VoteTypeId` (INTEGER) - The type of vote. Specifically `2` for UpMod and `3` for DownMod in this table. See the [Vote Types reference](vote_types.md) for a comprehensive list.
- `CreationDate` (TIMESTAMP) - The date and time the feedback was recorded.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_history_type_ids.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post History Type IDs
description: Enumerated types for post history events on Stack Exchange.
tags:
- post history
- enum
- moderation
- schema
- data dump
timestamp: '2026-05-28T23:33:44+00:00'
---

This document defines the enumerated types for various events recorded in the `PostHistory` table, tracking changes and moderation actions on posts across Stack Exchange sites.

- `1`: Initial Title - initial title (questions only)
- `2`: Initial Body - initial post raw body text
- `3`: Initial Tags - initial list of tags (questions only)
- `4`: Edit Title - modified title (questions only)
- `5`: Edit Body - modified post body (raw markdown)
- `6`: Edit Tags - modified list of tags (questions only)
- `7`: Rollback Title - reverted title (questions only)
- `8`: Rollback Body - reverted body (raw markdown)
- `9`: Rollback Tags - reverted list of tags (questions only)
- `10`: Post Closed - post voted to be closed
- `11`: Post Reopened - post voted to be reopened
- `12`: Post Deleted - post voted to be removed
- `13`: Post Undeleted - post voted to be restored
- `14`: Post Locked - post locked by moderator
- `15`: Post Unlocked - post unlocked by moderator
- `16`: Community Owned - post now community owned
- `17`: Post Migrated - post migrated - now replaced by 35/36 (away/here)
- `18`: Question Merged - question merged with deleted question
- `19`: Question Protected - question was protected by a moderator.
- `20`: Question Unprotected - question was unprotected by a moderator.
- `21`: Post Disassociated - OwnerUserId removed from post by admin
- `22`: Question Unmerged - answers/votes restored to previously merged question
- `24`: Suggested Edit Applied
- `25`: Post Tweeted
- `31`: Comment discussion moved to chat
- `33`: Post notice added - `comment` contains foreign key to PostNotices
- `34`: Post notice removed - `comment` contains foreign key to PostNotices
- `35`: Post migrated away - replaces id 17
- `36`: Post migrated here - replaces id 17
- `37`: Post merge source
- `38`: Post merge destination
- `50`: Bumped by Community User
- `52`: Question became hot network question (main) / Hot Meta question (meta)
- `53`: Question removed from hot network/meta questions by a moderator
- `66`: Created from Ask Wizard

## Older Dumps (Likely no longer present)
- `23`: Unknown dev related event
- `26`: Vote nullification by dev (ERM?)
- `27`: Post unmigrated/hidden moderator migration?
- `28`: Unknown suggestion event
- `29`: Unknown moderator event (possibly de-wikification?)
- `30`: Unknown event (too rare to guess)

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_notices.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Notices
description: Stores information about notices applied to posts, including their type,
  dates, and associated users.
tags:
- notices
- posts
- moderation
- schema
- data dump
timestamp: '2026-05-28T23:34:29+00:00'
---

This table stores information about notices applied to posts, such as "Citation needed" or "Duplicate".

## Schema
- `Id` (INTEGER) - Unique identifier for the post notice.
- `PostId` (INTEGER) - The ID of the post the notice is applied to. Links to the `posts` tables (`posts_questions` or `posts_answers`).
- `PostNoticeTypeId` (INTEGER) - The type of post notice. See the [Post Notice Types reference](post_notice_types.md) for possible values.
- `CreationDate` (TIMESTAMP) - The date and time the notice was created.
- `DeletionDate` (TIMESTAMP) - The date and time the notice was deleted. Nullable.
- `ExpiryDate` (TIMESTAMP) - The date and time the notice is set to expire. Nullable.
- `Body` (STRING) - When present, contains the custom text shown with the notice. Nullable.
- `OwnerUserId` (INTEGER) - The ID of the user who created the notice. Links to the `users` table. Nullable.
- `DeletionUserId` (INTEGER) - The ID of the user who deleted the notice. Links to the `users` table. Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_notice_class_ids.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Notice Class IDs
description: Enumerated class identifiers for different types of post notices.
tags:
- notices
- classes
- enum
- schema
- data dump
timestamp: '2026-05-28T23:34:34+00:00'
---

This document defines the enumerated class identifiers for different types of post notices.

- `0`: DMCA or Redditted
- `1`: Historical lock
- `2`: Bounty
- `4`: Moderator notice (lock)

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_notice_duration_ids.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Notice Duration IDs
description: Enumerated duration identifiers for post notices.
tags:
- notices
- duration
- enum
- schema
- data dump
timestamp: '2026-05-28T23:34:37+00:00'
---

This document defines the enumerated duration identifiers for post notices.

- `-1`: No duration specified
- `1`: 7 days (bounty)

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_notice_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Notice Types
description: Enumerated types for notices that can be applied to posts on Stack Exchange.
tags:
- notices
- posts
- enum
- schema
- data dump
timestamp: '2026-05-28T23:34:22+00:00'
---

This document defines the enumerated types for notices that can be applied to posts on Stack Exchange sites, as found in the `PostNotices` table.

- `1`: Citation needed
- `2`: Current event
- `3`: Insufficient explanation
- `10`: Current answers are outdated
- `11`: Draw attention
- `12`: Improve details
- `13`: Authoritative reference needed
- `14`: Canonical answer required
- `15`: Reward existing answer
- `20`: Content dispute
- `21`: Offtopic comments
- `22`: Historical significance
- `23`: Wiki Answer
- `24`: Policy Lock (SO Collectives)
- `25`: Recommended Answer (SO Collectives)
- `26`: Posted by Recognized Member/Admin (SO Collectives)
- `27`: Endorsed Edit (SO Collectives)
- `28`: Obsolete (SO Collectives)
- `1000`: Redditted (SO)
- `9001`: DMCA Takedown

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_notice_type_definitions.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Notice Type Definitions
description: Defines the structure and properties of different types of post notices.
tags:
- notices
- types
- definitions
- schema
- data dump
timestamp: '2026-05-28T23:34:42+00:00'
---

This table defines the structure and properties of different types of post notices that can be applied to posts.

## Schema
- `Id` (INTEGER) - Unique identifier for the post notice type.
- `ClassId` (INTEGER) - The class of the notice. See the [Post Notice Class IDs reference](post_notice_class_ids.md) for possible values.
- `Name` (STRING) - The name of the post notice type.
- `Body` (STRING) - Contains the default notice text. Nullable.
- `IsHidden` (BOOLEAN) - Indicates if the notice type is hidden.
- `Predefined` (BOOLEAN) - Indicates if the notice type is predefined.
- `PostNoticeDurationId` (INTEGER) - The duration identifier for the notice. See the [Post Notice Duration IDs reference](post_notice_duration_ids.md) for possible values.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\post_type_ids.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Post Type IDs
description: Enumerated IDs for different types of posts on Stack Exchange.
tags:
- post
- types
- enum
- stackoverflow
timestamp: '2026-05-28T23:31:12+00:00'
---

# Post Type IDs

This document defines the enumerated IDs for different types of posts found in the Stack Exchange data dump and SEDE.

- `1`: Question
- `2`: Answer
- `3`: Orphaned Tag Wiki
- `4`: Tag Wiki
- `5`: Tag Wiki Excerpt
- `6`: Moderator Nomination
- `7`: Privilege Wiki
- `8`: Status Wiki
- `10`: Deleted Question (not in public data dump)
- `11`: Section
- `12`: Article

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\related_questions.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Related Questions
description: Provides a mapping between questions and other related questions based
  on various factors.
tags:
- questions
- related
- schema
- data dump
timestamp: '2026-05-28T23:34:15+00:00'
---

This table provides a mapping between questions and other related questions, based on factors like content similarity or user behavior.

## Schema
- `PostId` (INTEGER) - The ID of the primary question. Links to the `posts_questions` table.
- `RelatedPostId` (INTEGER) - The ID of a related question. Links to the `posts_questions` table.
- `Position` (INTEGER) - The position or ranking of the related question.
- `Score` (INTEGER) - A score indicating the relevance or strength of the relationship.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_rejection_reasons.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Rejection Reasons
description: Defines canned rejection reasons for suggested edits.
tags:
- review
- rejection
- reasons
- suggested edits
- schema
- data dump
timestamp: '2026-05-28T23:34:58+00:00'
---

This table defines canned rejection reasons for suggested edits.

## Schema
- `Id` (INTEGER) - Unique identifier for the rejection reason.
- `Name` (STRING) - The name of the rejection reason.
- `Description` (STRING) - A detailed description of the rejection reason.
- `PostTypeId` (INTEGER) - The type of post the reason applies to (e.g., `5` for Wiki, `6` for Excerpt). Otherwise, it is null. Links to the [Post Type IDs reference](post_type_ids.md).

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_tasks.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Tasks
description: Stores information about moderation review tasks on Stack Exchange, such
  as suggested edits or close votes.
tags:
- review
- tasks
- moderation
- schema
- data dump
timestamp: '2026-05-28T23:35:29+00:00'
---

This table stores information about moderation review tasks on Stack Exchange, such as suggested edits or close votes.

## Schema
- `Id` (INTEGER) - Unique identifier for the review task.
- `ReviewTaskTypeId` (INTEGER) - The type of review task. See the [Review Task Types reference](review_task_types.md) for possible values.
- `CreationDate` (TIMESTAMP) - The date when the review task was created. Time data is typically removed to protect user privacy.
- `DeletionDate` (TIMESTAMP) - The date when the review task was deleted. Time data is typically removed to protect user privacy. Nullable.
- `ReviewTaskStateId` (INTEGER) - The current state of the review task. See the [Review Task States reference](review_task_states.md) for possible values.
- `PostId` (INTEGER) - The ID of the post associated with the review task. Links to the `posts` tables (`posts_questions` or `posts_answers`). Nullable.
- `SuggestedEditId` (INTEGER) - The ID of the suggested edit associated with the review task, if applicable. Links to the `SuggestedEdits` table. Nullable.
- `CompletedByReviewTaskId` (INTEGER) - The ID associated with the `ReviewTaskResult` that stores the outcome of a completed review. Links to the [Review Task Results reference](review_task_results.md). Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_task_results.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Task Results
description: Stores the outcomes and details of completed review tasks.
tags:
- review
- tasks
- results
- schema
- data dump
timestamp: '2026-05-28T23:35:11+00:00'
---

This table stores the outcomes and details of completed review tasks, such as approvals, rejections, or edits.

## Schema
- `Id` (INTEGER) - Unique identifier for the review task result.
- `ReviewTaskId` (INTEGER) - The ID of the review task this result is for. Links to the `ReviewTasks` table.
- `ReviewTaskResultTypeId` (INTEGER) - The type of result. See the [Review Task Result Types reference](review_task_result_types.md) for possible values.
- `CreationDate` (TIMESTAMP) - The date when the result was recorded. Time data is purposefully removed to protect user privacy.
- `RejectionReasonId` (INTEGER) - The ID of the rejection reason, specifically for suggested edits. Links to the [Review Rejection Reasons reference](review_rejection_reasons.md). Nullable.
- `Comment` (STRING) - A comment associated with the review result. Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_task_result_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Task Result Types
description: Enumerated types for the outcomes of review tasks on Stack Exchange.
tags:
- review
- tasks
- results
- enum
- schema
- data dump
timestamp: '2026-05-28T23:35:05+00:00'
---

This document defines the enumerated types for the outcomes of various review tasks on Stack Exchange sites.

- `1`: Skip
- `2`: Approve (suggested edits)
- `3`: Reject (suggested edits)
- `4`: Delete (low quality)
- `5`: Edit (first posts, late answers, low quality)
- `6`: Close (close, low quality)
- `7`: Looks OK (low quality)
- `8`: Do Not Close (close)
- `9`: Recommend Deletion (low quality answer)
- `10`: Recommend Close (low quality question)
- `11`: Other Action (first posts), previously "I\'m Done"
- `12`: Reopen (reopen)
- `13`: Leave Closed (reopen)
- `14`: Edit and Reopen (reopen)
- `15`: Excellent (community evaluation)
- `16`: Satisfactory (community evaluation)
- `17`: Needs Improvement (community evaluation)
- `18`: No Action Needed (first posts, late answers)

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_task_states.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Task States
description: Enumerated states for review tasks on Stack Exchange.
tags:
- review
- tasks
- states
- enum
- schema
- data dump
timestamp: '2026-05-28T23:35:21+00:00'
---

This document defines the enumerated states for review tasks on Stack Exchange sites.

- `1`: Active
- `2`: Completed
- `3`: Invalidated

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\review_task_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Review Task Types
description: Enumerated types for various review tasks on Stack Exchange.
tags:
- review
- tasks
- enum
- schema
- data dump
timestamp: '2026-05-28T23:35:18+00:00'
---

This document defines the enumerated types for various review tasks available on Stack Exchange sites.

- `1`: Suggested Edit
- `2`: Close Votes
- `3`: Low Quality Posts
- `4`: First Post
- `5`: Late Answer
- `6`: Reopen Vote
- `7`: Community Evaluation
- `8`: Link Validation
- `9`: Flagged Posts
- `10`: Triage
- `11`: Helper
- `12`: First Questions
- `13`: First Answers

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_databases.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE Databases
description: Details the structure of the internal SEDE database metadata table.
tags:
- SEDE
- internal
- databases
- schema
- data dump
timestamp: '2026-05-28T23:36:12+00:00'
---

This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each Stack Exchange site\'s database within SEDE.

## Schema
- `database_id` (INTEGER) - Unique identifier for the database.
- `database_name` (STRING) - The name of the database.
- `site_name` (STRING) - The short name of the site.
- `tiny_name` (STRING) - A very short name for the site.
- `long_name` (STRING) - The full name of the site.
- `site_type` (STRING) - Type of site, either \'main_site\' or \'meta_site\'.
- `site_url` (STRING) - The base URL of the Stack Exchange site.
- `sede_url` (STRING) - The URL to the site\'s data in SEDE.
- `api_site_parameter` (STRING) - The parameter used for the site in the Stack Exchange API.
- `initialized` (TIMESTAMP) - Datetime when populating that database started.
- `made_available` (TIMESTAMP) - Datetime when the database came online and was ready for queries.
- `processing_time` (STRING) - The duration between `initialized` and `made_available`, in `hh:mm:ss.fff` format.
- `questions` (INTEGER) - Total number of questions in this site at the time of the current refresh.
- `answers` (INTEGER) - Total number of answers in this site at the time of the current refresh.
- `latest_post` (TIMESTAMP) - The timestamp of the last non-deleted post captured in this refresh.
- `notes` (STRING) - This will be non-NULL when a database is in transition. Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_databases_markdown.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE Databases Markdown
description: Describes a SEDE internal table containing markdown content for database
  information.
tags:
- SEDE
- internal
- markdown
- schema
- data dump
timestamp: '2026-05-28T23:36:16+00:00'
---

This internal SEDE table contains markdown content related to database information, likely for display purposes.

## Schema
- `rn` (INTEGER) - Row number, used for sorting by the \'long name\' of the database.
- `content` (STRING) - Markdown content, potentially including headers, for database descriptions.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_ineachdb.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE In Each DB Stored Procedure
description: Describes the SEDE internal stored procedure for running SQL commands
  across multiple databases.
tags:
- SEDE
- internal
- stored procedure
- SQL
- data dump
timestamp: '2026-05-28T23:36:39+00:00'
---

This is an internal stored procedure in the Stack Exchange Data Explorer (SEDE) used to execute SQL commands across multiple databases (i.e., Stack Exchange sites).

## Parameters
| Parameter           | Data Type      | Default | Description                                                               |
|---------------------|----------------|---------|---------------------------------------------------------------------------|
| `@SQLCommand`       | `nvarchar(4000)` |         | The SQL statement to run.                                                 |
| `@IncludeMainSites` | `bit`          | `1`     | Set to `1` to include all non-meta sites.                                 |
| `@IncludeMetaSites` | `bit`          | `1`     | Set to `1` to include all meta sites.                                     |
| `@IncludeMainMeta`  | `bit`          | `1`     | Set to `1` to include `StackExchange.Meta`.                               |
| `@CollectResultsForMe` | `bit`          | `1`     | For standard `SELECT` queries, this will attempt to put each database's results into a `#temp` table. |
| `@ErrorOnSkippedSites` | `bit`          | `0`     | Set this to `1` if you want execution to halt in the event any site is missing due to transition change. |

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_sites.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE Sites
description: Details the structure of the internal SEDE sites metadata table.
tags:
- SEDE
- internal
- sites
- schema
- data dump
timestamp: '2026-05-28T23:36:26+00:00'
---

This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each Stack Exchange site.

## Schema
- `site_name` (STRING) - The short name of the site.
- `site_url` (STRING) - The base URL of the Stack Exchange site.
- `database_name` (STRING) - The name of the corresponding database in SEDE.
- `long_name` (STRING) - The full name of the site.
- `site_id` (INTEGER) - Unique identifier for the site.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_tables.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE Tables
description: Details the structure of the internal SEDE tables metadata table.
tags:
- SEDE
- internal
- tables
- schema
- data dump
timestamp: '2026-05-28T23:36:22+00:00'
---

This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each table within SEDE databases.

## Schema
- `database_id` (INTEGER) - The ID of the database, originating from `sys.databases`.
- `database_name` (STRING) - The name of the database, originating from `sys.databases`.
- `table_name` (STRING) - The name of the table, originating from `sys.tables`.
- `latest_date` (TIMESTAMP) - The latest timestamp (e.g., `CreationDate`) for tables that have such a column. Nullable.
- `row_count` (INTEGER) - The number of rows in the table, according to `sys.partitions`. For `Posts`, this includes `PostsWithDeleted`; for non-deleted `Posts`, refer to `answers + questions` from `dbo.sede_databases`.
- `initialized` (TIMESTAMP) - Timestamp of initial table creation.
- `made_available` (TIMESTAMP) - Timestamp of the final operation against the table.
- `processing_time` (STRING) - Duration of `initialized` -> `made_available`, in `{hh:mm:ss.fff}` format.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\sede_users.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: SEDE Users
description: Details the structure of the internal SEDE users metadata table.
tags:
- SEDE
- internal
- users
- schema
- data dump
timestamp: '2026-05-28T23:36:32+00:00'
---

This internal table in the Stack Exchange Data Explorer (SEDE) provides user-specific metadata across sites.

## Schema
- `account_id` (INTEGER) - The global Stack Exchange Network account ID.
- `site_id` (INTEGER) - The ID of the site the user is associated with. Links to `references/sede_sites.md`.
- `user_id` (INTEGER) - The local user ID on that specific site. Links to the `users` table.
- `reputation` (INTEGER) - The user\'s reputation on that specific site.
- `question_count` (INTEGER) - The number of questions asked by the user on that specific site.
- `answer_count` (INTEGER) - The number of answers provided by the user on that specific site.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\suggested_edits.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Suggested Edits
description: Stores information about suggested edits to posts that are awaiting review.
tags:
- suggested edits
- posts
- schema
- data dump
timestamp: '2026-05-28T23:35:39+00:00'
---

This table stores information about suggested edits to posts that are awaiting review. If both `ApprovalDate` and `RejectionDate` are null, then the edit is still in review.

## Schema
- `Id` (INTEGER) - Unique identifier for the suggested edit.
- `PostId` (INTEGER) - The ID of the post to which the suggested edit applies. Links to the `posts` tables (`posts_questions` or `posts_answers`).
- `CreationDate` (TIMESTAMP) - The date and time the suggested edit was created.
- `ApprovalDate` (TIMESTAMP) - The date and time the suggested edit was approved. Null if not yet approved.
- `RejectionDate` (TIMESTAMP) - The date and time the suggested edit was rejected. Null if not yet rejected.
- `OwnerUserId` (INTEGER) - The ID of the user who suggested the edit. Links to the `users` table.
- `Comment` (STRING) - A comment provided with the suggested edit. Nullable.
- `Text` (STRING) - The proposed new body text for the post. Nullable.
- `Title` (STRING) - The proposed new title for the post. Nullable.
- `Tags` (STRING) - The proposed new tags for the post. Nullable.
- `RevisionGUID` (STRING) - A GUID for the revision.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\suggested_edit_votes.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Suggested Edit Votes
description: Records votes cast on suggested edits, indicating approval or rejection.
tags:
- suggested edits
- votes
- schema
- data dump
timestamp: '2026-05-28T23:35:46+00:00'
---

This table records votes cast on suggested edits, indicating approval or rejection.

## Schema
- `Id` (INTEGER) - Unique identifier for the suggested edit vote.
- `SuggestedEditId` (INTEGER) - The ID of the suggested edit that was voted on. Links to the [Suggested Edits reference](suggested_edits.md).
- `UserId` (INTEGER) - The ID of the user who cast the vote. Links to the `users` table.
- `VoteTypeId` (INTEGER) - The type of vote. Specifically `2` for Approve (UpMod) and `3` for Reject (DownMod). See the [Vote Types reference](vote_types.md) for a comprehensive list.
- `CreationDate` (TIMESTAMP) - The date and time the vote was cast.
- `TargetUserId` (INTEGER) - The ID of the target user for the vote (e.g., the user whose reputation changed). Links to the `users` table. Nullable.
- `TargetRepChange` (INTEGER) - The change in reputation for the target user as a result of this vote. Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\tag_synonyms.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Tag Synonyms
description: Stores relationships between synonymous tags on Stack Exchange.
tags:
- tags
- synonyms
- schema
- data dump
timestamp: '2026-05-28T23:35:52+00:00'
---

This table stores relationships between synonymous tags, allowing for automatic renaming or suggestion of alternative tags.

## Schema
- `Id` (INTEGER) - Unique identifier for the tag synonym.
- `SourceTagName` (STRING) - The name of the tag that is considered a synonym. Nullable. Links to the `tags` table\'s `tag_name` field.
- `TargetTagName` (STRING) - The name of the canonical tag to which `SourceTagName` refers. Nullable. Links to the `tags` table\'s `tag_name` field.
- `CreationDate` (TIMESTAMP) - The date and time the tag synonym was created.
- `OwnerUserId` (INTEGER) - The ID of the user who created the tag synonym. Nullable. Links to the `users` table.
- `AutoRenameCount` (INTEGER) - The number of times this synonym has resulted in an automatic tag rename.
- `LastAutoRename` (TIMESTAMP) - The date and time of the last automatic rename using this synonym. Nullable.
- `Score` (INTEGER) - A score indicating the quality or consensus of the synonym.
- `ApprovedByUserId` (INTEGER) - The ID of the user who approved the tag synonym. Nullable. Links to the `users` table.
- `ApprovalDate` (TIMESTAMP) - The date and time the tag synonym was approved. Nullable.

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\references\vote_types.md
================================================
---
type: Reference
resource: https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede
title: Vote Types
description: Enumerated types for votes on posts and other entities across Stack Exchange
  sites.
tags:
- votes
- enum
- moderation
- schema
- data dump
timestamp: '2026-05-28T23:33:26+00:00'
---

This document defines the enumerated types for various votes that can occur on posts and other entities across Stack Exchange sites.

- `-1`: InformModerator
- `0`: UndoMod
- `1`: AcceptedByOriginator
- `2`: UpMod (Upvote)
- `3`: DownMod (Downvote)
- `4`: Offensive
- `5`: Favorite (Bookmark) - feature removed after October 2022, replaced by Saves.
- `6`: Close - Close votes are only stored in the `PostHistory` table after 2013-06-25.
- `7`: Reopen
- `8`: BountyStart
- `9`: BountyClose
- `10`: Deletion
- `11`: Undeletion
- `12`: Spam
- `15`: ModeratorReview - a moderator looking at a flagged post.
- `16`: ApproveEditSuggestion
- `17`: Reaction1 (Teams: celebrate)
- `18`: Helpful
- `19`: ThankYou
- `20`: WellWritten
- `21`: Follow
- `22`: Reaction2 (Teams: smile)
- `23`: Reaction3 (Teams: mind blown)
- `24`: Reaction4 (Teams: clap)
- `25`: Reaction5 (Teams: heart)
- `26`: Reaction6 (Teams: fire)
- `27`: Reaction7 (Teams: trophy)
- `28`: Reaction8 (Teams: wave)
- `29`: Outdated
- `30`: NotOutdated
- `31`: PreVote
- `32`: CollectiveDiscussionUpvote
- `33`: CollectiveDiscussionDownvote (no longer used)
- `35`: privateAiAnswerCorrect
- `36`: privateAiAnswerIncorrect
- `37`: privateAiAnswerPartiallyCorrect

# Citations
- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\badges.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/badges
title: Badges
description: This table contains information about badges awarded to users on the
  Stack Overflow platform, including details about the badge name, class, and whether
  it is tag-based.
tags:
- stackoverflow
- badges
- users
- awards
- schema
- data dump
timestamp: '2026-05-28T23:32:46+00:00'
---

The `badges` table records all badges awarded to users on the Stack Overflow Q&A platform. Each row represents a single badge awarded, detailing its unique identifier, name, the date it was awarded, the ID of the user who received it, its class (e.g., gold, silver, bronze), and whether it is a tag-based badge. The data spans from October 26, 2016, up to November 24, 2022. This table can be joined with the [users](users.md) table on `user_id` to get more information about the badge recipients.

# Schema

- `id` (INTEGER) - Unique identifier for the badge award.
- `name` (STRING) - Name of the badge.
- `date` (TIMESTAMP) - Timestamp when the badge was awarded (e.g., `2008-09-15T08:55:03.923`).
- `user_id` (INTEGER) - Identifier of the user who received the badge.
- `class` (INTEGER) - The class of the badge. See the [Badge Classes reference](../references/badge_classes.md) for possible values.
- `tag_based` (BOOLEAN) - `True` if the badge is for a tag, otherwise it is a named badge.

# Common query patterns

```sql
SELECT
    name,
    COUNT(*) AS badge_count
  FROM
    `bigquery-public-data.stackoverflow.badges`
  GROUP BY
    name
  ORDER BY
    badge_count DESC
  LIMIT 5;
```

```sql
SELECT
    b.name,
    b.date
  FROM
    `bigquery-public-data.stackoverflow.badges` AS b
    JOIN `bigquery-public-data.stackoverflow.users` AS u ON b.user_id = u.id
  WHERE
    u.display_name = \'Jon Skeet\'
  ORDER BY
    b.date DESC;
```

```sql
SELECT
    DATE(date) AS award_date,
    COUNT(*) AS badges_awarded
  FROM
    `bigquery-public-data.stackoverflow.badges`
  GROUP BY
    award_date
  ORDER BY
    award_date DESC
  LIMIT 10;
```

# Citations

[1] [BigQuery Public Data: stackoverflow.badges](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/badges)
[2] [Stack Overflow](https://stackoverflow.com/)
[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\comments.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/comments
title: Comments
description: This table contains comments made by users on posts within the Stack
  Overflow platform, including details about the comment text, score, and author,
  along with licensing information.
tags:
- stackoverflow
- comments
- community
- user-generated content
- schema
- data dump
timestamp: '2026-05-28T23:32:34+00:00'
---

The `comments` table in the [Stack Overflow dataset](../datasets/stackoverflow.md) captures all user-generated comments on various posts across the platform. Each row in this table represents a unique comment, providing details such as the comment\'s text, its creation timestamp, and a score reflecting its community reception. Comments are linked to the specific [posts](posts_questions.md) (questions or answers) they belong to via `post_id` and to the [users](users.md) who authored them via `user_id`. This table is crucial for analyzing user engagement, content quality feedback, and conversational threads on the platform.

# Schema
- `id` (INTEGER) - Unique identifier for the comment.
- `post_id` (INTEGER) - The ID of the post (question or answer) to which the comment belongs.
- `score` (INTEGER) - The score of the comment, reflecting upvotes and downvotes.
- `text` (STRING) - The content of the comment (Comment body).
- `creation_date` (TIMESTAMP) - The date and time the comment was posted.
- `user_display_name` (STRING) - The display name of the user who made the comment. Nullable.
- `user_id` (INTEGER) - The ID of the user who made the comment. Optional, absent if user has been deleted.
- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.

# Common query patterns
```sql
-- Retrieve the 10 most recent comments
SELECT
  id,
  text,
  creation_date,
  user_display_name
FROM
  `bigquery-public-data.stackoverflow.comments`
ORDER BY
  creation_date DESC
LIMIT 10;
```

```sql
-- Count comments per user
SELECT
  user_display_name,
  COUNT(id) AS total_comments
FROM
  `bigquery-public-data.stackoverflow.comments`
WHERE
  user_display_name IS NOT NULL
GROUP BY
  user_display_name
ORDER BY
  total_comments DESC
LIMIT 10;
```

```sql
-- Find comments with a high score for a specific post
SELECT
  id,
  text,
  score,
  creation_date
FROM
  `bigquery-public-data.stackoverflow.comments`
WHERE
  post_id = 45651 AND score > 5
ORDER BY
  score DESC;
```

# Citations
[1] [BigQuery Public Dataset: Stack Overflow Comments](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/comments)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\index.md
================================================
# BigQuery Table

* [Badges](badges.md) - This table contains information about badges awarded to users on the Stack Overflow platform, including details about the badge name, class, and whether it is tag-based.
* [Comments](comments.md) - This table contains comments made by users on posts within the Stack Overflow platform, including details about the comment text, score, and author, along with licensing information.
* [Orphaned Tag Wiki Posts](posts_orphaned_tag_wiki.md) - This table contains Tag Wiki posts that are no longer associated with an active tag on Stack Overflow.
* [Post History](post_history.md) - Tracks the revision history of posts on Stack Overflow, including details about event types, associated users, and content changes.
* [Post Links](post_links.md) - Contains information about links between posts on Stack Overflow, including the type of link and the posts involved.
* [Posts Answers](posts_answers.md) - Contains information about answers to questions posted on Stack Overflow, including detailed schema fields, their descriptions, and licensing information.
* [Posts Moderator Nomination](posts_moderator_nomination.md) - Contains posts where users nominate themselves or others to become moderators on the Stack Overflow platform.
* [Posts Privilege Wiki](posts_privilege_wiki.md) - This table contains posts describing various privileges on Stack Overflow, such as editing questions and retagging.
* [Posts Tag Wiki](posts_tag_wiki.md) - Contains detailed wiki descriptions for tags on Stack Overflow.
* [Posts Tag Wiki Excerpt](posts_tag_wiki_excerpt.md) - Contains excerpts from tag wikis on Stack Overflow.
* [Posts Wiki Placeholder](posts_wiki_placeholder.md) - Placeholder table for Stack Overflow wiki posts, containing community-maintained help and information.
* [Stack Overflow Posts (Legacy)](stackoverflow_posts.md) - A legacy table containing all posts from Stack Overflow. This table is deprecated; use `posts_answers`, `posts_questions`, or other `posts_*` tables instead.
* [Stack Overflow Questions](posts_questions.md) - This table contains information about all question posts on Stack Overflow, including detailed schema fields, their descriptions, and licensing information.
* [Tags](tags.md) - A table containing information about tags used on Stack Overflow, including tag names, counts, and references to their excerpt and wiki posts, along with moderation settings.
* [Users](users.md) - This table contains information about users registered on Stack Overflow, including detailed profile information, activity metrics, and network-wide identifiers.
* [Votes](votes.md) - This table contains information about votes cast on posts within the Stack Overflow platform, including the vote type, associated post and user, and bounty details.


================================================
FILE: okf\bundles\stackoverflow\tables\posts_answers.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_answers
title: Posts Answers
description: Contains information about answers to questions posted on Stack Overflow,
  including detailed schema fields, their descriptions, and licensing information.
tags:
- stackoverflow
- posts
- answers
- community
- programming
- schema
- data dump
timestamp: '2026-05-28T23:32:11+00:00'
---

The `posts_answers` table in the [stackoverflow](../datasets/stackoverflow.md) dataset contains all posts identified as answers to questions on the Stack Overflow platform. Each row in this table represents a single answer, providing details such as the answer\'s body, its score, creation date, and the ID of the parent question it answers. This table can be joined with other tables like [posts_questions](posts_questions.md) on `parent_id` to link answers to their respective questions. It\'s a valuable resource for analyzing answer quality, user contributions, and engagement patterns within the Stack Overflow community.

# Schema
- `id` (INTEGER) - Unique identifier for the answer.
- `title` (STRING) - The title of the post. This is typically NULL for answers.
- `body` (STRING) - The body of the post, as rendered HTML, not Markdown.
- `accepted_answer_id` (INTEGER) - The ID of the answer that was accepted for a question. This field is only applicable to questions, so it is typically NULL for answers.
- `answer_count` (INTEGER) - The number of answers. This field is only applicable to questions, so it is typically NULL for answers.
- `closed_date` (TIMESTAMP) - The date and time the post was closed; present only if the post is closed. This is typically NULL for answers.
- `comment_count` (INTEGER) - The number of comments.
- `community_owned_date` (TIMESTAMP) - The date and time the post became community wiki\'d.
- `creation_date` (TIMESTAMP) - The date and time the answer was created.
- `favorite_count` (INTEGER) - The number of times the post has been favorited. Nullable. This is typically NULL for answers.
- `last_activity_date` (TIMESTAMP) - The datetime of the post\'s most recent activity.
- `last_edit_date` (TIMESTAMP) - The date and time of the most recent edit to the answer.
- `last_editor_display_name` (STRING) - The display name of the last editor. Nullable.
- `last_editor_user_id` (INTEGER) - The ID of the last editor user. Nullable.
- `owner_display_name` (STRING) - The display name of the answer\'s owner. Nullable.
- `owner_user_id` (INTEGER) - The ID of the answer\'s owner. Present only if user has not been deleted.
- `parent_id` (INTEGER) - The ID of the question this answer belongs to. Present only if `PostTypeId = 2`.
- `post_type_id` (INTEGER) - The type of post. For answers, this is `2`. See the [Post Type IDs reference](../references/post_type_ids.md) for possible values.
- `score` (INTEGER) - The score of the answer; generally non-zero for Questions, Answers, and Moderator Nominations.
- `tags` (STRING) - Tags associated with the post. This is typically NULL for answers, as tags are associated with the parent question.
- `view_count` (INTEGER) - The number of times the post was viewed. This field is only applicable to questions, so it is typically NULL for answers.
- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.

# Common query patterns
1. Get the 10 most highly-scored answers in a specific date range:
```sql
SELECT
  id,
  score,
  creation_date,
  body
FROM
  `bigquery-public-data.stackoverflow.posts_answers`
WHERE
  creation_date BETWEEN \'2023-01-01\' AND \'2023-01-31\'
ORDER BY
  score DESC
LIMIT 10
```
2. Find the total number of answers by a specific user:
```sql
SELECT
  owner_user_id,
  COUNT(id) AS total_answers
FROM
  `bigquery-public-data.stackoverflow.posts_answers`
WHERE
  owner_user_id = 12345 -- Replace with a valid user ID
GROUP BY
  owner_user_id
```
3. Join answers with their corresponding questions:
```sql
SELECT
  q.title AS question_title,
  a.body AS answer_body,
  a.score AS answer_score
FROM
  `bigquery-public-data.stackoverflow.posts_answers` AS a
JOIN
  `bigquery-public-data.stackoverflow.posts_questions` AS q
ON
  a.parent_id = q.id
WHERE
  q.id = 5000 -- Example question ID
```

# Citations
[1] [Stack Overflow Posts Answers Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_answers)
[2] [Stack Overflow Public Data Explorer](https://data.stackexchange.com/)
[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_moderator_nomination.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_moderator_nomination
title: Posts Moderator Nomination
description: Contains posts where users nominate themselves or others to become moderators
  on the Stack Overflow platform.
tags: stackoverflow, posts, moderator, nomination
timestamp: '2026-05-28T23:27:37+00:00'
---

This table contains posts submitted by users nominating themselves or others to become moderators on the Stack Overflow platform. Each row represents a single moderator nomination post, detailing the candidate's qualifications or reasons for nomination.

The `creation_date` field indicates when the nomination post was created, providing a historical record of moderator elections and community engagement.

# Schema

- `id`: Unique identifier for the post.
- `title`: Title of the nomination post (often null for this post type).
- `body`: The content of the nomination post, describing the candidate's qualifications.
- `accepted_answer_id`: (STRING) ID of the accepted answer, if applicable.
- `answer_count`: (STRING) Number of answers to the post.
- `comment_count`: Number of comments on the post.
- `community_owned_date`: Timestamp when the post became community-owned.
- `creation_date`: Timestamp when the post was created.
- `favorite_count`: (STRING) Number of times the post has been favorited.
- `last_activity_date`: Timestamp of the last activity on the post.
- `last_edit_date`: Timestamp of the last edit to the post.
- `last_editor_display_name`: Display name of the last editor.
- `last_editor_user_id`: User ID of the last editor.
- `owner_display_name`: Display name of the post owner.
- `owner_user_id`: User ID of the post owner. This links to the [/users](users.md) table.
- `parent_id`: (STRING) ID of the parent post, if this is a reply.
- `post_type_id`: Type of the post (e.g., 6 for moderator nomination).
- `score`: The score of the post.
- `tags`: (STRING) Tags associated with the post.
- `view_count`: (STRING) Number of views the post has received.

# Common query patterns

```sql
SELECT
    id,
    creation_date,
    owner_user_id,
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_moderator_nomination`
  WHERE
    creation_date BETWEEN '2021-01-01' AND '2021-12-31'
  ORDER BY
    creation_date DESC
  LIMIT 100
```

```sql
SELECT
    p.id,
    p.creation_date,
    u.display_name AS nominator_name,
    p.body
  FROM
    `bigquery-public-data.stackoverflow.posts_moderator_nomination` AS p
  JOIN
    `bigquery-public-data.stackoverflow.users` AS u
    ON p.owner_user_id = u.id
  WHERE
    p.creation_date >= '2022-01-01'
  LIMIT 10
```

# Citations

[1] [BigQuery Public Data: Stack Overflow Posts Moderator Nomination](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_moderator_nomination)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_orphaned_tag_wiki.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_orphaned_tag_wiki
title: Orphaned Tag Wiki Posts
description: This table contains Tag Wiki posts that are no longer associated with
  an active tag on Stack Overflow.
tags: stackoverflow, posts, tag, wiki, orphaned
timestamp: '2026-05-28T23:27:56+00:00'
---

This table, part of the public [Stack Overflow dataset](../datasets/stackoverflow.md), contains posts that were originally created as Tag Wikis but are no longer linked to an existing tag within the Stack Overflow platform. These 'orphaned' Tag Wiki entries typically describe the purpose and usage of a specific programming language, technology, or concept. Each row represents a single Tag Wiki post, identified by a unique `id`. The `post_type_id` for these entries is typically `3`. The table includes the body of the wiki, creation dates, and metadata about the last editor and activity. It can be used to analyze historical Tag Wiki content that has become disassociated from its original tag.

# Schema

- `id`: Unique identifier for the post.
- `title`: Title of the post (often null for Tag Wikis).
- `body`: The main content of the Tag Wiki, often including HTML.
- `accepted_answer_id`: (Always null for this post type).
- `answer_count`: (Always null for this post type).
- `comment_count`: Number of comments on the post.
- `community_owned_date`: Timestamp when the post became community-owned.
- `creation_date`: Timestamp when the post was created.
- `favorite_count`: Number of times the post has been favorited.
- `last_activity_date`: Timestamp of the last activity on the post.
- `last_edit_date`: Timestamp of the last edit to the post.
- `last_editor_display_name`: Display name of the last user to edit the post.
- `last_editor_user_id`: User ID of the last user to edit the post.
- `owner_display_name`: Display name of the user who owns the post.
- `owner_user_id`: User ID of the user who owns the post.
- `parent_id`: (Always null for this post type).
- `post_type_id`: Type of the post. For this table, it's typically `3` (Tag Wiki).
- `score`: The score of the post.
- `tags`: Tags associated with the post (often null as these are orphaned).
- `view_count`: Number of times the post has been viewed.

# Common query patterns

```sql
SELECT
    id,
    creation_date,
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`
  WHERE
    creation_date < '2015-01-01'
  LIMIT 100;
```

```sql
SELECT
    id,
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`
  WHERE
    id = 4164933;
```

```sql
SELECT
    count(id) AS total_orphaned_tag_wikis
  FROM
    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`;
```

# Citations

[1] [BigQuery Public Data: Stack Overflow posts_orphaned_tag_wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_orphaned_tag_wiki)
[2] [Stack Overflow](https://stackoverflow.com/)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_privilege_wiki.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_privilege_wiki
title: Posts Privilege Wiki
description: This table contains posts describing various privileges on Stack Overflow,
  such as editing questions and retagging.
tags: stackoverflow, posts, wiki, privileges, community, documentation
timestamp: '2026-05-28T23:28:24+00:00'
---

This BigQuery table, `posts_privilege_wiki`, contains posts that describe various user privileges within the Stack Overflow community. These entries serve as informational guides, explaining what a privilege entails, when and how to use it, and related community guidelines. Examples include details on editing questions and answers, or the process of retagging questions. Each row represents a specific privilege explanation, detailing its purpose and usage, contributing to the self-documentation of the platform's community features.

# Schema
- `id`: Unique identifier for the post.
- `title`: The title of the post. For these privilege wiki entries, titles are often null, with the main content in the body.
- `body`: The full content of the privilege explanation, often including HTML formatting.
- `accepted_answer_id`: (Always null for these types of posts)
- `answer_count`: (Always null for these types of posts)
- `comment_count`: Number of comments on the post.
- `community_owned_date`: Date when the post became community-owned.
- `creation_date`: Timestamp when the post was created.
- `favorite_count`: (Always null for these types of posts)
- `last_activity_date`: Timestamp of the last activity on the post.
- `last_edit_date`: Timestamp of the last edit to the post.
- `last_editor_display_name`: Display name of the last user who edited the post.
- `last_editor_user_id`: User ID of the last user who edited the post.
- `owner_display_name`: Display name of the post's owner.
- `owner_user_id`: User ID of the post's owner. Often -1 for community posts.
- `parent_id`: (Always null for these types of posts)
- `post_type_id`: Type of post, typically 8 for wiki entries.
- `score`: The score (upvotes minus downvotes) of the post.
- `tags`: (Always null for these types of posts)
- `view_count`: (Always null for these types of posts)

# Common query patterns
```sql
SELECT
  id,
  body
FROM
  `bigquery-public-data.stackoverflow.posts_privilege_wiki`
WHERE
  creation_date >= '2013-01-01'
LIMIT 100
```
```sql
SELECT
  id,
  body,
  creation_date
FROM
  `bigquery-public-data.stackoverflow.posts_privilege_wiki`
WHERE
  body LIKE '%edit questions and answers%'
```

# Citations
[1] [BigQuery Public Dataset: Stack Overflow Posts Privilege Wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_privilege_wiki)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_questions.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_questions
title: Stack Overflow Questions
description: This table contains information about all question posts on Stack Overflow,
  including detailed schema fields, their descriptions, and licensing information.
tags:
- Stack Overflow
- questions
- posts
- programming
- community
- schema
- data dump
timestamp: '2026-05-28T23:31:54+00:00'
---

This table, part of the public [Stack Overflow dataset](../datasets/stackoverflow.md), contains comprehensive data for every question asked on the Stack Overflow platform. Each row represents a unique question post, including its content, metadata such as creation and last activity dates, view counts, and score. It serves as a primary resource for analyzing user activity, popular topics, and the overall dynamics of the Q&A community. The table can be joined with other tables like [posts_answers](posts_answers.md) to link questions to their corresponding answers, or with [users](users.md) to retrieve information about the question\'s author.

# Schema
The schema contains fields related to the question\'s content, status, and associated user data. Key fields include `id` for unique identification, `title` and `body` for the question\'s content, `creation_date` for when the question was posted, and `tags` for categorizing the question.

- `id` (INTEGER) - Unique identifier for the post.
- `title` (STRING) - The title of the question (for `PostTypeId = 1`).
- `body` (STRING) - The body of the post, as rendered HTML, not Markdown.
- `accepted_answer_id` (INTEGER) - The ID of the accepted answer; present only if `PostTypeId = 1`.
- `answer_count` (INTEGER) - The number of undeleted answers; present only if `PostTypeId = 1`.
- `closed_date` (TIMESTAMP) - The date and time the post was closed; present only if the post is closed.
- `comment_count` (INTEGER) - The number of comments.
- `community_owned_date` (TIMESTAMP) - The date and time the post became community wiki\'d.
- `creation_date` (TIMESTAMP) - The date and time the post was created.
- `favorite_count` (INTEGER) - The number of times the post has been favorited. Nullable.
- `last_activity_date` (TIMESTAMP) - The datetime of the post\'s most recent activity.
- `last_edit_date` (TIMESTAMP) - The date and time of the most recent edit to the post.
- `last_editor_display_name` (STRING) - The display name of the last editor. Nullable.
- `last_editor_user_id` (INTEGER) - The ID of the last editor user. Nullable.
- `owner_display_name` (STRING) - The display name of the post owner. Nullable.
- `owner_user_id` (INTEGER) - The ID of the post owner. Present only if user has not been deleted; always -1 for tag wiki entries.
- `parent_id` (STRING) - Parent ID. For questions (`PostTypeId = 1`), this field is typically NULL. It is mainly used for answers (`PostTypeId = 2`) to link to their parent question.
- `post_type_id` (INTEGER) - The type of the post. For questions, this is `1`. See the [Post Type IDs reference](../references/post_type_ids.md) for possible values.
- `score` (INTEGER) - The score of the post; generally non-zero for Questions, Answers, and Moderator Nominations.
- `tags` (STRING) - The tags associated with the question (for `PostTypeId = 1`).
- `view_count` (INTEGER) - The number of times the post has been viewed. Nullable.
- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.

# Common query patterns
1. **Find the most viewed questions in a specific tag:**
   ```sql
   SELECT
     title,
     view_count
   FROM
     `bigquery-public-data.stackoverflow.posts_questions`
   WHERE
     tags LIKE \'%<your-tag>%'
   ORDER BY
     view_count DESC
   LIMIT 10
   ```
2. **Count questions posted per year:**
   ```sql
   SELECT
     EXTRACT(YEAR FROM creation_date) AS year,
     COUNT(id) AS question_count
   FROM
     `bigquery-public-data.stackoverflow.posts_questions`
   GROUP BY
     year
   ORDER BY
     year DESC
   ```
3. **Get questions with accepted answers and their answer count:**
   ```sql
   SELECT
     id,
     title,
     accepted_answer_id,
     answer_count
   FROM
     `bigquery-public-data.stackoverflow.posts_questions`
   WHERE
     accepted_answer_id IS NOT NULL
   ORDER BY
     answer_count DESC
   LIMIT 5
   ```

# Citations
[1] [posts_questions](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_questions)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_tag_wiki.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki
title: Posts Tag Wiki
description: Contains detailed wiki descriptions for tags on Stack Overflow.
tags: stackoverflow, tag, wiki, posts, bigquery
timestamp: '2026-05-28T23:29:03+00:00'
---

The `posts_tag_wiki` table in the BigQuery Stack Overflow dataset stores detailed wiki descriptions for various [tags](tags.md) used on the platform. Each row represents a wiki entry associated with a specific tag, providing comprehensive information about its purpose, usage, and related concepts. This table is useful for understanding the context and definition of tags, which helps in categorizing questions and answers accurately. The `body` field contains the rich text content of the wiki. This table is a valuable resource for anyone analyzing the structure and content organization of [Stack Overflow](../datasets/stackoverflow.md). The grain of this table is one row per tag wiki entry.

# Schema
- `id`: INTEGER, Unique identifier for the tag wiki post.
- `title`: STRING, Title of the tag wiki (often null for actual tag wiki posts).
- `body`: STRING, The main content of the tag wiki, containing detailed description.
- `accepted_answer_id`: STRING, ID of the accepted answer, if applicable.
- `answer_count`: STRING, Number of answers.
- `comment_count`: INTEGER, Number of comments on the post.
- `community_owned_date`: TIMESTAMP, Date when the post became community-owned.
- `creation_date`: TIMESTAMP, Date and time the post was created.
- `favorite_count`: STRING, Number of times the post has been favorited.
- `last_activity_date`: TIMESTAMP, Date and time of the last activity on the post.
- `last_edit_date`: TIMESTAMP, Date and time of the last edit to the post.
- `last_editor_display_name`: STRING, Display name of the last editor.
- `last_editor_user_id`: INTEGER, User ID of the last editor.
- `owner_display_name`: STRING, Display name of the post owner.
- `owner_user_id`: INTEGER, User ID of the post owner.
- `parent_id`: STRING, ID of the parent post, if applicable.
- `post_type_id`: INTEGER, Identifier for the type of post (e.g., 5 for Tag Wiki).
- `score`: INTEGER, The score of the post.
- `tags`: STRING, Tags associated with the post (usually null for tag wiki posts, as the wiki *is* for a tag).
- `view_count`: STRING, Number of views the post has received.

# Common query patterns

To retrieve the body of a specific tag wiki:
```sql
SELECT
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki`
  WHERE
    id = 5046395
```

To find tag wiki entries created after a certain date:
```sql
SELECT
    id,
    creation_date,
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki`
  WHERE
    creation_date > '2020-01-01'
  LIMIT 100
```

To count the number of tag wiki entries per creation year:
```sql
SELECT
    EXTRACT(YEAR FROM creation_date) AS creation_year,
    COUNT(id) AS number_of_wikis
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki`
  GROUP BY
    creation_year
  ORDER BY
    creation_year DESC
```

# Citations
[1] [BigQuery Public Dataset: Stack Overflow Posts Tag Wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_tag_wiki_excerpt.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki_excerpt
title: Posts Tag Wiki Excerpt
description: Contains excerpts from tag wikis on Stack Overflow.
tags: stackoverflow, posts, tags, wiki, excerpt
timestamp: '2026-05-28T23:29:18+00:00'
---

The `posts_tag_wiki_excerpt` table in the BigQuery Stack Overflow public dataset contains summary information, or excerpts, from the wiki pages associated with various tags. Each row represents a single tag wiki excerpt, providing details such as the excerpt's body, its associated tag, and metadata about its creation and last modification. This table is useful for understanding the brief descriptions provided for tags on the Stack Overflow platform, offering quick insights into what a particular tag represents.

# Schema
- `id`: Unique identifier for the tag wiki excerpt.
- `title`: Title of the tag wiki excerpt.
- `body`: The content of the tag wiki excerpt.
- `accepted_answer_id`: (STRING)
- `answer_count`: (STRING)
- `comment_count`: Number of comments on the excerpt.
- `community_owned_date`: When the excerpt became community-owned.
- `creation_date`: When the excerpt was created.
- `favorite_count`: (STRING)
- `last_activity_date`: Last activity date on the excerpt.
- `last_edit_date`: Last edit date of the excerpt.
- `last_editor_display_name`: Display name of the last editor.
- `last_editor_user_id`: User ID of the last editor.
- `owner_display_name`: Display name of the owner.
- `owner_user_id`: User ID of the owner.
- `parent_id`: (STRING)
- `post_type_id`: Type of post.
- `score`: Score of the excerpt.
- `tags`: The tag associated with this wiki excerpt (e.g., `<python>`).
- `view_count`: (STRING)

# Common query patterns

```sql
SELECT
    title,
    body
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`
  WHERE
    id = 12345
```

```sql
SELECT
    title,
    tags
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`
  WHERE
    body LIKE '%python%'
```

```sql
SELECT
    title,
    creation_date,
    last_editor_display_name
  FROM
    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`
  ORDER BY
    creation_date DESC
  LIMIT 10
```

# Citations
[1] [BigQuery Public Data: Stack Overflow posts_tag_wiki_excerpt](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki_excerpt)


================================================
FILE: okf\bundles\stackoverflow\tables\posts_wiki_placeholder.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_wiki_placeholder
title: Posts Wiki Placeholder
description: Placeholder table for Stack Overflow wiki posts, containing community-maintained
  help and information.
tags: stackoverflow, posts, wiki, help, community, documentation
timestamp: '2026-05-28T23:29:41+00:00'
---

This table serves as a placeholder for various wiki posts within the Stack Overflow community, containing community-maintained help, guidelines, and other informational content. Each row in this table represents a single wiki post, identified by a unique `id`. These posts often include details about election processes, help center articles, and general community guidelines.

# Schema

- `id`: INTEGER, Unique identifier for the wiki post.
- `title`: STRING, Title of the wiki post.
- `body`: STRING, The main content of the wiki post.
- `accepted_answer_id`: STRING
- `answer_count`: STRING
- `comment_count`: INTEGER, Number of comments on the post.
- `community_owned_date`: STRING
- `creation_date`: TIMESTAMP, Timestamp when the post was created.
- `favorite_count`: STRING
- `last_activity_date`: TIMESTAMP, Timestamp of the last activity on the post.
- `last_edit_date`: TIMESTAMP, Timestamp when the post was last edited.
- `last_editor_display_name`: STRING
- `last_editor_user_id`: INTEGER, User ID of the last editor.
- `owner_display_name`: STRING
- `owner_user_id`: INTEGER, User ID of the post owner.
- `parent_id`: STRING
- `post_type_id`: INTEGER, Type of the post (e.g., 7 for Wiki).
- `score`: INTEGER, Score of the post.
- `tags`: STRING, Tags associated with the post.
- `view_count`: STRING

# Common query patterns

```sql
SELECT
    id, title, body
  FROM
    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`
  WHERE
    id = 8041931
```

```sql
SELECT
    id, title
  FROM
    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`
  WHERE
    body LIKE '%election process%'
  LIMIT 100
```

```sql
SELECT
    EXTRACT(YEAR FROM creation_date) AS creation_year,
    COUNT(id) AS post_count
  FROM
    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`
  GROUP BY
    creation_year
  ORDER BY
    creation_year DESC
```

# Citations

[1] [BigQuery Public Data: Stack Overflow posts_wiki_placeholder](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_wiki_placeholder)


================================================
FILE: okf\bundles\stackoverflow\tables\post_history.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_history
title: Post History
description: Tracks the revision history of posts on Stack Overflow, including details
  about event types, associated users, and content changes.
tags:
- stackoverflow
- post history
- revisions
- edits
- schema
- data dump
timestamp: '2026-05-28T23:33:54+00:00'
---

This table, part of the [Stack Overflow dataset](../datasets/stackoverflow.md), records every historical event related to posts, such as edits, rollbacks, and changes in post type. Each row represents a single historical event for a particular post, identified by `post_id`, with the event\'s timestamp recorded in `creation_date`. This allows for a comprehensive audit trail of how content on Stack Overflow evolves over time.

### Schema

- `id` (INTEGER) - Unique identifier for the post history entry.
- `post_history_type_id` (INTEGER) - An integer representing the type of history event. See the [Post History Type IDs reference](../references/post_history_type_ids.md) for possible values.
- `post_id` (INTEGER) - The ID of the post to which this history event belongs. Links to `posts_questions` or `posts_answers` tables.
- `revision_guid` (STRING) - A GUID that groups multiple history records caused by a single action.
- `creation_date` (TIMESTAMP) - The timestamp when the history event occurred (e.g., `2009-03-05T22:28:34.823`).
- `user_id` (INTEGER) - The ID of the user who performed the action, if applicable. Links to the `users` table.
- `user_display_name` (STRING) - Populated if a user has been removed and is no longer referenced by `user_id`; also happens to the author of a migrated post. Nullable.
- `comment` (STRING) - A brief comment describing the history event. This field can also contain specific IDs based on `PostHistoryTypeId`:
    - If `PostHistoryTypeId = 10`, it contains the `CloseReasonId`. See the [Close Reason Types reference](../references/close_reason_types.md).
    - If `PostHistoryTypeId` is `33` or `34`, it contains the `PostNoticeId`.
- `text` (STRING) - A raw version of the new value for a given revision. This field can also contain specific JSON encoded strings based on `PostHistoryTypeId`:
    - If `PostHistoryTypeId` is `10, 11, 12, 13, 14, 15, 19, 20, 35`, it contains a JSON encoded string with all users who have voted for that `PostHistoryTypeId`.
    - If it is a duplicate close vote, the JSON string contains an array of original questions as `OriginalQuestionIds`.
    - If `PostHistoryTypeId = 17`, it contains migration details (`from <url>` or `to <url>`).
- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.

### Common query patterns

```sql
SELECT
  creation_date,
  post_history_type_id,
  user_id,
  comment,
  text
FROM
  `bigquery-public-data.stackoverflow.post_history`
WHERE
  post_id = 456789 -- Example post ID
ORDER BY
  creation_date DESC
LIMIT 10;
```

```sql
SELECT
  post_history_type_id,
  COUNT(*) AS event_count
FROM
  `bigquery-public-data.stackoverflow.post_history`
GROUP BY
  post_history_type_id
ORDER BY
  event_count DESC;
```

```sql
SELECT
  t2.display_name,
  COUNT(t1.id) AS edit_count
FROM
  `bigquery-public-data.stackoverflow.post_history` AS t1
INNER JOIN
  `bigquery-public-data.stackoverflow.users` AS t2
ON
  t1.user_id = t2.id
WHERE
  t1.post_history_type_id = 2 -- Assuming \'2\' means \'Post Edited\'
GROUP BY
  t2.display_name
ORDER BY
  edit_count DESC
LIMIT 5;
```

### Citations

[1] [BigQuery Public Dataset: Stack Overflow Post History](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_history)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\post_links.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_links
title: Post Links
description: Contains information about links between posts on Stack Overflow, including
  the type of link and the posts involved.
tags:
- stackoverflow
- posts
- links
- schema
- data dump
timestamp: '2026-05-28T23:34:10+00:00'
---

The `post_links` table in the [stackoverflow](../datasets/stackoverflow.md) dataset records relationships between posts on the Stack Overflow platform. Each row represents a single link between two posts, identifying the `post_id` and `related_post_id`. This table can be used to understand how discussions evolve, identify duplicate questions, or find related content across the platform. The `link_type_id` categorizes the nature of the relationship, such as "Linked" or "Duplicate".

# Schema

*   `id` (INTEGER) - Unique identifier for the post link (primary key).
*   `creation_date` (TIMESTAMP) - The timestamp when the link was created.
*   `link_type_id` (INTEGER) - The type of link. See the [Link Types reference](../references/link_types.md) for possible values.
*   `post_id` (INTEGER) - The ID of the source post. Links to the `id` in the `posts_questions` or `posts_answers` tables.
*   `related_post_id` (INTEGER) - The ID of the target/related post. Links to the `id` in the `posts_questions` or `posts_answers` tables.

# Common query patterns

```sql
SELECT
  pl.post_id,
  pl.related_post_id,
  pl.creation_date,
  pl.link_type_id
FROM
  `bigquery-public-data.stackoverflow.post_links` AS pl
WHERE
  pl.post_id = 12345
LIMIT 100;
```

```sql
SELECT
  link_type_id,
  COUNT(*) AS link_count
FROM
  `bigquery-public-data.stackoverflow.post_links`
GROUP BY
  link_type_id
ORDER BY
  link_count DESC;
```

# Citations

[1] [BigQuery Public Dataset: Stack Overflow Post Links](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_links)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\stackoverflow_posts.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/stackoverflow_posts
title: Stack Overflow Posts (Legacy)
description: A legacy table containing all posts from Stack Overflow. This table is
  deprecated; use `posts_answers`, `posts_questions`, or other `posts_*` tables instead.
tags: Stack Overflow, posts, legacy, deprecated
timestamp: '2026-05-28T23:30:00+00:00'
---

This table, `stackoverflow_posts`, is a legacy table from the [stackoverflow](../datasets/stackoverflow.md) dataset in Google Cloud's BigQuery public datasets. It contains all posts (questions and answers) from Stack Overflow. **This table is deprecated and should not be used for new queries.** Users are advised to use the more specific and updated tables like [posts_questions](posts_questions.md) for questions and [posts_answers](posts_answers.md) for answers. Each row in this table represents a single post, with fields such as `id`, `title`, `body`, `creation_date`, and user information. The table contains approximately 31 million rows and over 31 GB of data.

# Schema

- `id`: Unique identifier for the post.
- `title`: The title of the post (for questions).
- `body`: The main content of the post.
- `accepted_answer_id`: The ID of the accepted answer (for questions).
- `answer_count`: Number of answers for a question.
- `comment_count`: Number of comments on the post.
- `community_owned_date`: Timestamp when the post became community-owned.
- `creation_date`: Timestamp when the post was created.
- `favorite_count`: Number of times the post has been favorited.
- `last_activity_date`: Timestamp of the last activity on the post.
- `last_edit_date`: Timestamp of the last edit.
- `last_editor_display_name`: Display name of the last editor.
- `last_editor_user_id`: User ID of the last editor.
- `owner_display_name`: Display name of the post owner.
- `owner_user_id`: User ID of the post owner.
- `parent_id`: For answers, the ID of the question it answers.
- `post_type_id`: Type of the post (e.g., 1 for question, 2 for answer).
- `score`: The score of the post (upvotes - downvotes).
- `tags`: Comma-separated tags associated with the post (for questions).
- `view_count`: Number of views for the post (for questions).

# Common query patterns

```sql
SELECT
  id,
  title,
  creation_date
FROM
  `bigquery-public-data.stackoverflow.stackoverflow_posts`
WHERE
  creation_date BETWEEN '2008-01-01' AND '2008-01-31'
  AND post_type_id = 1 -- Questions only
LIMIT 100;
```

# Citations

[1] [BigQuery Console - stackoverflow_posts](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/stackoverflow_posts)
[2] [BigQuery Public Datasets - Stack Overflow](https://cloud.google.com/bigquery/public-data/stackoverflow)


================================================
FILE: okf\bundles\stackoverflow\tables\tags.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/tags
title: Tags
description: A table containing information about tags used on Stack Overflow, including
  tag names, counts, and references to their excerpt and wiki posts, along with moderation
  settings.
tags:
- stackoverflow
- tags
- community
- metadata
- schema
- data dump
timestamp: '2026-05-28T23:34:53+00:00'
---

This table, `tags`, from the [stackoverflow](../datasets/stackoverflow.md) dataset, provides a comprehensive list of all tags used across the Stack Overflow platform. Each row represents a unique tag, detailing its name, the total count of times it has been used, and references to its corresponding excerpt and wiki posts. This table is essential for understanding the topics and technologies discussed on Stack Overflow, allowing for analysis of tag popularity, trends, and content organization.

# Schema
The `tags` table contains the following fields:
- `id` (INTEGER) - Unique identifier for the tag.
- `tag_name` (STRING) - The name of the tag (e.g., \'python\', \'java\', \'javascript\').
- `count` (INTEGER) - The total number of times this tag has been used.
- `excerpt_post_id` (INTEGER) - The ID of the Post that holds the excerpt text of the tag. Nullable.
- `wiki_post_id` (INTEGER) - The ID of the Post that holds the wiki text of the tag. Nullable.
- `is_moderator_only` (BOOLEAN) - Indicates if the tag can only be used by moderators.
- `is_required` (BOOLEAN) - Indicates if the tag is required for certain post types.

# Common query patterns

1. Find the most popular tags:
```sql
SELECT
    tag_name,
    count
  FROM
    `bigquery-public-data.stackoverflow.tags`
  GROUP BY
    tag_name
  ORDER BY
    count DESC
  LIMIT 10
```

2. Get details for a specific tag:
```sql
SELECT
    id,
    tag_name,
    count,
    excerpt_post_id,
    wiki_post_id
  FROM
    `bigquery-public-data.stackoverflow.tags`
  WHERE
    tag_name = \'python\'
```

3. Join with posts to find posts related to a popular tag (example uses `posts_questions` which is related to `stackoverflow_posts`):
```sql
SELECT
    p.title,
    t.tag_name
  FROM
    `bigquery-public-data.stackoverflow.posts_questions` AS p,
    `bigquery-public-data.stackoverflow.tags` AS t
  WHERE
    p.tags LIKE CONCAT(\'%<\', t.tag_name, \'>%\' )
    AND t.tag_name = \'javascript\'
  LIMIT 5
```

# Citations
[1] [BigQuery Public Dataset: Stack Overflow Tags](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/tags)
[2] [Stack Overflow Dataset on BigQuery](https://console.cloud.google.com/marketplace/details/stackoverflow/stackoverflow)
[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\users.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/users
title: Users
description: This table contains information about users registered on Stack Overflow,
  including detailed profile information, activity metrics, and network-wide identifiers.
tags:
- Stack Overflow
- users
- profiles
- community
- schema
- data dump
timestamp: '2026-05-28T23:32:24+00:00'
---

This table, `users`, from the [stackoverflow](../datasets/stackoverflow.md) dataset, stores profiles of registered users on the Stack Overflow platform. Each row represents a unique user, identified by their `id`. The table includes details such as display name, creation date, last access date, reputation, and vote counts. It provides insights into the activity and characteristics of the Stack Overflow community members.

# Schema

*   `id` (INTEGER) - Unique identifier for the user.
*   `display_name` (STRING) - The publicly visible name of the user.
*   `about_me` (STRING) - User-provided free-form text about themselves. Nullable.
*   `age` (INTEGER) - User-provided age. Nullable.
*   `account_id` (INTEGER) - User\'s Stack Exchange Network profile ID; NULL if the user has hidden this community in their profile. Nullable.
*   `creation_date` (TIMESTAMP) - Timestamp when the user account was created.
*   `email_hash` (STRING) - Gravatar email hash, now always NULL and will not appear as an attribute in the data dump XML. Nullable.
*   `last_access_date` (TIMESTAMP) - Datetime user last loaded a page; updated every 30 minutes at most.
*   `location` (STRING) - User-provided geographical location. Nullable.
*   `reputation` (INTEGER) - The user\'s reputation score.
*   `up_votes` (INTEGER) - How many upvotes the user has cast.
*   `down_votes` (INTEGER) - Total number of downvotes received by the user.
*   `views` (INTEGER) - Number of times the user\'s profile has been viewed.
*   `profile_image_url` (STRING) - URL of the user\'s profile picture. Nullable.
*   `website_url` (STRING) - URL of the user\'s personal website. Nullable.

# Common query patterns

1.  Find the top 10 users by reputation:
    ```sql
    SELECT
      id,
      display_name,
      reputation
    FROM
      `bigquery-public-data.stackoverflow.users`
    ORDER BY
      reputation DESC
    LIMIT 10
    ```
2.  Count users created per year:
    ```sql
    SELECT
      EXTRACT(YEAR FROM creation_date) AS creation_year,
      COUNT(id) AS user_count
    FROM
      `bigquery-public-data.stackoverflow.users`
    GROUP BY
      creation_year
    ORDER BY
      creation_year
    ```
3.  Find users with a high number of upvotes given:
    ```sql
    SELECT
      display_name,
      up_votes
    FROM
      `bigquery-public-data.stackoverflow.users`
    WHERE
      up_votes > 1000
    ORDER BY
      up_votes DESC
    LIMIT 5
    ```

# Citations

[1] [Stack Overflow Users Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/users)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\tables\votes.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/votes
title: Votes
description: This table contains information about votes cast on posts within the
  Stack Overflow platform, including the vote type, associated post and user, and
  bounty details.
tags:
- votes
- posts
- stackoverflow
- community
- engagement
- schema
- data dump
timestamp: '2026-05-28T23:36:03+00:00'
---

This table, `votes`, from the [stackoverflow](../datasets/stackoverflow.md) dataset, records individual voting events on various posts (questions, answers, etc.) within the Stack Overflow platform. Each row represents a single vote, providing details such as the unique vote identifier, the timestamp of the vote, the ID of the post that received the vote, and the type of vote (e.g., upvote, downvote, favorite). This table is crucial for analyzing user engagement, post popularity, and the overall quality assessment mechanisms within the Stack Overflow community. The grain of this table is one row per vote event, and it captures historical voting data from its creation date up to the last modification.

# Schema

- `id` (INTEGER) - Unique identifier for the vote.
- `post_id` (INTEGER) - The ID of the post to which the vote was cast. Links to `posts_questions` or `posts_answers` tables.
- `vote_type_id` (INTEGER) - The type of vote. See the [Vote Types reference](../references/vote_types.md) for a comprehensive list.
- `user_id` (INTEGER) - The ID of the user who cast the vote. Present only if `VoteTypeId` is `5` (Favorite/Bookmark) or `8` (BountyStart); -1 if user is deleted. Nullable. Links to the `users` table.
- `creation_date` (TIMESTAMP) - The date when the vote was cast. Time data is purposefully removed to protect user privacy.
- `bounty_amount` (INTEGER) - The amount of bounty associated with the vote. Present only if `VoteTypeId` is `8` (BountyStart) or `9` (BountyClose). Nullable.

# Common query patterns

```sql
-- Get the total number of votes for each vote type
SELECT
  vote_type_id,
  COUNT(id) AS total_votes
FROM
  `bigquery-public-data.stackoverflow.votes`
GROUP BY
  vote_type_id
ORDER BY
  total_votes DESC;
```

```sql
-- Find the top 10 most voted posts by summing up all votes (assuming all vote_type_ids contribute positively)
SELECT
  post_id,
  COUNT(id) AS total_votes
FROM
  `bigquery-public-data.stackoverflow.votes`
GROUP BY
  post_id
ORDER BY
  total_votes DESC
LIMIT 10;
```

```sql
-- Count votes over time
SELECT
  DATE(creation_date) AS vote_date,
  COUNT(id) AS daily_votes
FROM
  `bigquery-public-data.stackoverflow.votes`
WHERE
  creation_date >= \'2020-01-01\' AND creation_date < \'2021-01-01\'
GROUP BY
  vote_date
ORDER BY
  vote_date;
```

# Citations
[1] [BigQuery Public Dataset: Stack Overflow Votes Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/votes)
[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)


================================================
FILE: okf\bundles\stackoverflow\viz.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>OKF Bundle Viewer</title>
<script src="https://cdn.jsdelivr.net/npm/cytoscape@3.28.1/dist/cytoscape.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script>
<style>
* { box-sizing: border-box; }
body {
  margin: 0;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
  font-size: 14px;
  color: #0f172a;
  background: #f8fafc;
  display: flex;
  flex-direction: column;
  height: 100vh;
}
header {
  display: flex;
  align-items: center;
  justify-content: space-between;
  padding: 10px 16px;
  background: #fff;
  border-bottom: 1px solid #e2e8f0;
  flex-shrink: 0;
}
.title strong { font-size: 16px; margin-right: 8px; }
.muted { color: #64748b; font-size: 12px; }
.controls { display: flex; gap: 8px; }
.controls input, .controls select, .controls button {
  font-size: 13px;
  padding: 5px 8px;
  border: 1px solid #cbd5e1;
  border-radius: 4px;
  background: #fff;
}
.controls input { width: 220px; }
.controls button { cursor: pointer; background: #f1f5f9; }
.controls button:hover { background: #e2e8f0; }

main {
  display: flex;
  flex: 1;
  min-height: 0;
}
#graph {
  flex: 1 1 60%;
  background: #fff;
  border-right: 1px solid #e2e8f0;
  min-width: 0;
  position: relative;
}
#detail {
  flex: 0 0 40%;
  overflow-y: auto;
  padding: 18px 22px;
  background: #fff;
}
#detail-empty {
  text-align: center;
  margin-top: 40px;
}

.detail-header { margin-bottom: 12px; }
.detail-header h1 {
  font-size: 18px;
  margin: 4px 0 2px;
  font-weight: 600;
}
.type-chip {
  display: inline-block;
  padding: 2px 8px;
  border-radius: 10px;
  font-size: 11px;
  font-weight: 600;
  color: #fff;
  background: #94a3b8;
  text-transform: uppercase;
  letter-spacing: 0.5px;
}
dl.frontmatter {
  display: grid;
  grid-template-columns: 90px 1fr;
  row-gap: 4px;
  column-gap: 12px;
  margin: 8px 0 12px;
  font-size: 13px;
}
dl.frontmatter dt {
  color: #64748b;
  font-weight: 500;
}
dl.frontmatter dd { margin: 0; }
dl.frontmatter a { color: #2563eb; word-break: break-all; }

.tag {
  display: inline-block;
  padding: 1px 6px;
  margin: 0 4px 2px 0;
  border-radius: 4px;
  background: #f1f5f9;
  color: #475569;
  font-size: 11px;
}

hr { border: none; border-top: 1px solid #e2e8f0; margin: 14px 0; }

#detail-body { font-size: 13px; line-height: 1.55; }
#detail-body h1 {
  font-size: 16px; margin: 18px 0 6px;
  padding-bottom: 4px; border-bottom: 1px solid #e2e8f0;
}
#detail-body h2 { font-size: 14px; margin: 14px 0 4px; }
#detail-body h3 { font-size: 13px; margin: 12px 0 4px; }
#detail-body p { margin: 6px 0; }
#detail-body code {
  background: #f1f5f9;
  padding: 1px 4px;
  border-radius: 3px;
  font-size: 12px;
  font-family: ui-monospace, "SF Mono", Consolas, monospace;
}
#detail-body pre {
  background: #0f172a;
  color: #e2e8f0;
  padding: 10px 12px;
  border-radius: 6px;
  overflow-x: auto;
  font-size: 12px;
}
#detail-body pre code { background: transparent; color: inherit; padding: 0; }
#detail-body ul, #detail-body ol { padding-left: 22px; margin: 6px 0; }
#detail-body li { margin: 2px 0; }
#detail-body table { border-collapse: collapse; margin: 8px 0; }
#detail-body th, #detail-body td {
  border: 1px solid #e2e8f0; padding: 4px 8px; font-size: 12px;
}
#detail-body a.internal { color: #2563eb; cursor: pointer; }
#detail-body a.external { color: #2563eb; }

#detail-backlinks { margin-top: 18px; }
#detail-backlinks h2 { font-size: 13px; color: #64748b; margin-bottom: 6px; }
#detail-backlinks ul { padding-left: 18px; }
#detail-backlinks a { color: #2563eb; cursor: pointer; }

</style>
</head>
<body>
<header>
  <div class="title">
    <strong id="bundle-name"></strong>
    <span class="muted">OKF bundle</span>
  </div>
  <div class="controls">
    <input id="search" type="search" placeholder="Search title / id / tag">
    <select id="filter-type">
      <option value="">All types</option>
    </select>
    <select id="layout">
      <option value="cose">cose (force)</option>
      <option value="concentric">concentric</option>
      <option value="breadthfirst">breadth-first</option>
      <option value="circle">circle</option>
      <option value="grid">grid</option>
    </select>
    <button id="reset">Reset view</button>
  </div>
</header>

<main>
  <section id="graph"></section>
  <section id="detail">
    <div id="detail-empty" class="muted">Click a node to see its details.</div>
    <article id="detail-content" hidden>
      <header class="detail-header">
        <span class="type-chip" id="detail-type"></span>
        <h1 id="detail-title"></h1>
        <div class="muted" id="detail-id"></div>
      </header>
      <dl class="frontmatter">
        <dt>Description</dt><dd id="detail-description"></dd>
        <dt>Resource</dt><dd id="detail-resource"></dd>
        <dt>Tags</dt><dd id="detail-tags"></dd>
      </dl>
      <hr>
      <div id="detail-body"></div>
      <section id="detail-backlinks" hidden>
        <h2>Cited by</h2>
        <ul id="backlinks-list"></ul>
      </section>
    </article>
  </section>
</main>

<script>
window.BUNDLE_NAME = "stackoverflow";
window.BUNDLE = {"nodes": [{"data": {"id": "datasets/stackoverflow", "label": "Stack Overflow Public Dataset", "type": "BigQuery Dataset", "description": "The Stack Overflow public dataset contains a variety of tables related to Stack Overflow user activity, posts, and tags. This dataset is no longer actively updated.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow", "tags": ["Stack Overflow, public data, community, Q&A"], "color": "#8b5cf6", "size": 41}}, {"data": {"id": "references/badge_classes", "label": "Badge Classes", "type": "Reference", "description": "Enumerated classes for badges awarded on Stack Exchange sites.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["badges", "classes", "enum", "stackoverflow"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/close_as_off_topic_reason_types", "label": "Close As Off-Topic Reason Types", "type": "Reference", "description": "Defines the types and guidance for reasons why a post might be closed as off-topic.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["close reasons", "off-topic", "enum", "schema", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "references/close_reason_types", "label": "Close Reason Types", "type": "Reference", "description": "Enumerated types for reasons why a post might be closed on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["close reasons", "moderation", "enum", "schema", "data dump"], "color": "#10b981", "size": 33}}, {"data": {"id": "references/creative_commons_by_sa_4_0_license", "label": "Creative Commons Attribution-ShareAlike 4.0 International License", "type": "Reference", "description": "Details the attribution requirements for content licensed under CC BY-SA 4.0 for the Stack Exchange network.", "resource": "https://archive.org/details/stackexchange", "tags": ["licensing", "creative commons", "attribution", "data dump"], "color": "#10b981", "size": 38}}, {"data": {"id": "references/flag_types", "label": "Flag Types", "type": "Reference", "description": "Enumerated types for flags related to post moderation on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["flags", "moderation", "enum", "schema", "data dump"], "color": "#10b981", "size": 32}}, {"data": {"id": "references/link_types", "label": "Link Types", "type": "Reference", "description": "Enumerated types for links between posts on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["links", "enum", "schema", "data dump"], "color": "#10b981", "size": 32}}, {"data": {"id": "references/pending_flags", "label": "Pending Flags", "type": "Reference", "description": "Stores information about pending flags and votes related to post moderation.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["flags", "moderation", "schema", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "references/post_feedback", "label": "Post Feedback", "type": "Reference", "description": "Collects up and down votes from anonymous visitors and unregistered users.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["feedback", "votes", "anonymous", "schema", "data dump"], "color": "#10b981", "size": 34}}, {"data": {"id": "references/post_history_type_ids", "label": "Post History Type IDs", "type": "Reference", "description": "Enumerated types for post history events on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["post history", "enum", "moderation", "schema", "data dump"], "color": "#10b981", "size": 42}}, {"data": {"id": "references/post_notice_class_ids", "label": "Post Notice Class IDs", "type": "Reference", "description": "Enumerated class identifiers for different types of post notices.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["notices", "classes", "enum", "schema", "data dump"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/post_notice_duration_ids", "label": "Post Notice Duration IDs", "type": "Reference", "description": "Enumerated duration identifiers for post notices.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["notices", "duration", "enum", "schema", "data dump"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/post_notice_type_definitions", "label": "Post Notice Type Definitions", "type": "Reference", "description": "Defines the structure and properties of different types of post notices.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["notices", "types", "definitions", "schema", "data dump"], "color": "#10b981", "size": 34}}, {"data": {"id": "references/post_notice_types", "label": "Post Notice Types", "type": "Reference", "description": "Enumerated types for notices that can be applied to posts on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["notices", "posts", "enum", "schema", "data dump"], "color": "#10b981", "size": 34}}, {"data": {"id": "references/post_notices", "label": "Post Notices", "type": "Reference", "description": "Stores information about notices applied to posts, including their type, dates, and associated users.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["notices", "posts", "moderation", "schema", "data dump"], "color": "#10b981", "size": 36}}, {"data": {"id": "references/post_type_ids", "label": "Post Type IDs", "type": "Reference", "description": "Enumerated IDs for different types of posts on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["post", "types", "enum", "stackoverflow"], "color": "#10b981", "size": 32}}, {"data": {"id": "references/related_questions", "label": "Related Questions", "type": "Reference", "description": "Provides a mapping between questions and other related questions based on various factors.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["questions", "related", "schema", "data dump"], "color": "#10b981", "size": 33}}, {"data": {"id": "references/review_rejection_reasons", "label": "Review Rejection Reasons", "type": "Reference", "description": "Defines canned rejection reasons for suggested edits.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "rejection", "reasons", "suggested edits", "schema", "data dump"], "color": "#10b981", "size": 33}}, {"data": {"id": "references/review_task_result_types", "label": "Review Task Result Types", "type": "Reference", "description": "Enumerated types for the outcomes of review tasks on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "tasks", "results", "enum", "schema", "data dump"], "color": "#10b981", "size": 34}}, {"data": {"id": "references/review_task_results", "label": "Review Task Results", "type": "Reference", "description": "Stores the outcomes and details of completed review tasks.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "tasks", "results", "schema", "data dump"], "color": "#10b981", "size": 35}}, {"data": {"id": "references/review_task_states", "label": "Review Task States", "type": "Reference", "description": "Enumerated states for review tasks on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "tasks", "states", "enum", "schema", "data dump"], "color": "#10b981", "size": 31}}, {"data": {"id": "references/review_task_types", "label": "Review Task Types", "type": "Reference", "description": "Enumerated types for various review tasks on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "tasks", "enum", "schema", "data dump"], "color": "#10b981", "size": 32}}, {"data": {"id": "references/review_tasks", "label": "Review Tasks", "type": "Reference", "description": "Stores information about moderation review tasks on Stack Exchange, such as suggested edits or close votes.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["review", "tasks", "moderation", "schema", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "references/sede_databases", "label": "SEDE Databases", "type": "Reference", "description": "Details the structure of the internal SEDE database metadata table.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "databases", "schema", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "references/sede_databases_markdown", "label": "SEDE Databases Markdown", "type": "Reference", "description": "Describes a SEDE internal table containing markdown content for database information.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "markdown", "schema", "data dump"], "color": "#10b981", "size": 32}}, {"data": {"id": "references/sede_ineachdb", "label": "SEDE In Each DB Stored Procedure", "type": "Reference", "description": "Describes the SEDE internal stored procedure for running SQL commands across multiple databases.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "stored procedure", "SQL", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "references/sede_sites", "label": "SEDE Sites", "type": "Reference", "description": "Details the structure of the internal SEDE sites metadata table.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "sites", "schema", "data dump"], "color": "#10b981", "size": 33}}, {"data": {"id": "references/sede_tables", "label": "SEDE Tables", "type": "Reference", "description": "Details the structure of the internal SEDE tables metadata table.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "tables", "schema", "data dump"], "color": "#10b981", "size": 35}}, {"data": {"id": "references/sede_users", "label": "SEDE Users", "type": "Reference", "description": "Details the structure of the internal SEDE users metadata table.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["SEDE", "internal", "users", "schema", "data dump"], "color": "#10b981", "size": 34}}, {"data": {"id": "references/suggested_edit_votes", "label": "Suggested Edit Votes", "type": "Reference", "description": "Records votes cast on suggested edits, indicating approval or rejection.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["suggested edits", "votes", "schema", "data dump"], "color": "#10b981", "size": 35}}, {"data": {"id": "references/suggested_edits", "label": "Suggested Edits", "type": "Reference", "description": "Stores information about suggested edits to posts that are awaiting review.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["suggested edits", "posts", "schema", "data dump"], "color": "#10b981", "size": 36}}, {"data": {"id": "references/tag_synonyms", "label": "Tag Synonyms", "type": "Reference", "description": "Stores relationships between synonymous tags on Stack Exchange.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["tags", "synonyms", "schema", "data dump"], "color": "#10b981", "size": 36}}, {"data": {"id": "references/vote_types", "label": "Vote Types", "type": "Reference", "description": "Enumerated types for votes on posts and other entities across Stack Exchange sites.", "resource": "https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede", "tags": ["votes", "enum", "moderation", "schema", "data dump"], "color": "#10b981", "size": 37}}, {"data": {"id": "tables/badges", "label": "Badges", "type": "BigQuery Table", "description": "This table contains information about badges awarded to users on the Stack Overflow platform, including details about the badge name, class, and whether it is tag-based.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/badges", "tags": ["stackoverflow", "badges", "users", "awards", "schema", "data dump"], "color": "#3b82f6", "size": 40}}, {"data": {"id": "tables/comments", "label": "Comments", "type": "BigQuery Table", "description": "This table contains comments made by users on posts within the Stack Overflow platform, including details about the comment text, score, and author, along with licensing information.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/comments", "tags": ["stackoverflow", "comments", "community", "user-generated content", "schema", "data dump"], "color": "#3b82f6", "size": 41}}, {"data": {"id": "tables/post_history", "label": "Post History", "type": "BigQuery Table", "description": "Tracks the revision history of posts on Stack Overflow, including details about event types, associated users, and content changes.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_history", "tags": ["stackoverflow", "post history", "revisions", "edits", "schema", "data dump"], "color": "#3b82f6", "size": 47}}, {"data": {"id": "tables/post_links", "label": "Post Links", "type": "BigQuery Table", "description": "Contains information about links between posts on Stack Overflow, including the type of link and the posts involved.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_links", "tags": ["stackoverflow", "posts", "links", "schema", "data dump"], "color": "#3b82f6", "size": 38}}, {"data": {"id": "tables/posts_answers", "label": "Posts Answers", "type": "BigQuery Table", "description": "Contains information about answers to questions posted on Stack Overflow, including detailed schema fields, their descriptions, and licensing information.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_answers", "tags": ["stackoverflow", "posts", "answers", "community", "programming", "schema", "data dump"], "color": "#3b82f6", "size": 51}}, {"data": {"id": "tables/posts_moderator_nomination", "label": "Posts Moderator Nomination", "type": "BigQuery Table", "description": "Contains posts where users nominate themselves or others to become moderators on the Stack Overflow platform.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_moderator_nomination", "tags": ["stackoverflow, posts, moderator, nomination"], "color": "#3b82f6", "size": 42}}, {"data": {"id": "tables/posts_orphaned_tag_wiki", "label": "Orphaned Tag Wiki Posts", "type": "BigQuery Table", "description": "This table contains Tag Wiki posts that are no longer associated with an active tag on Stack Overflow.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_orphaned_tag_wiki", "tags": ["stackoverflow, posts, tag, wiki, orphaned"], "color": "#3b82f6", "size": 43}}, {"data": {"id": "tables/posts_privilege_wiki", "label": "Posts Privilege Wiki", "type": "BigQuery Table", "description": "This table contains posts describing various privileges on Stack Overflow, such as editing questions and retagging.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_privilege_wiki", "tags": ["stackoverflow, posts, wiki, privileges, community, documentation"], "color": "#3b82f6", "size": 41}}, {"data": {"id": "tables/posts_questions", "label": "Stack Overflow Questions", "type": "BigQuery Table", "description": "This table contains information about all question posts on Stack Overflow, including detailed schema fields, their descriptions, and licensing information.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_questions", "tags": ["Stack Overflow", "questions", "posts", "programming", "community", "schema", "data dump"], "color": "#3b82f6", "size": 51}}, {"data": {"id": "tables/posts_tag_wiki", "label": "Posts Tag Wiki", "type": "BigQuery Table", "description": "Contains detailed wiki descriptions for tags on Stack Overflow.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki", "tags": ["stackoverflow, tag, wiki, posts, bigquery"], "color": "#3b82f6", "size": 44}}, {"data": {"id": "tables/posts_tag_wiki_excerpt", "label": "Posts Tag Wiki Excerpt", "type": "BigQuery Table", "description": "Contains excerpts from tag wikis on Stack Overflow.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki_excerpt", "tags": ["stackoverflow, posts, tags, wiki, excerpt"], "color": "#3b82f6", "size": 40}}, {"data": {"id": "tables/posts_wiki_placeholder", "label": "Posts Wiki Placeholder", "type": "BigQuery Table", "description": "Placeholder table for Stack Overflow wiki posts, containing community-maintained help and information.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_wiki_placeholder", "tags": ["stackoverflow, posts, wiki, help, community, documentation"], "color": "#3b82f6", "size": 40}}, {"data": {"id": "tables/stackoverflow_posts", "label": "Stack Overflow Posts (Legacy)", "type": "BigQuery Table", "description": "A legacy table containing all posts from Stack Overflow. This table is deprecated; use `posts_answers`, `posts_questions`, or other `posts_*` tables instead.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/stackoverflow_posts", "tags": ["Stack Overflow, posts, legacy, deprecated"], "color": "#3b82f6", "size": 41}}, {"data": {"id": "tables/tags", "label": "Tags", "type": "BigQuery Table", "description": "A table containing information about tags used on Stack Overflow, including tag names, counts, and references to their excerpt and wiki posts, along with moderation settings.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/tags", "tags": ["stackoverflow", "tags", "community", "metadata", "schema", "data dump"], "color": "#3b82f6", "size": 42}}, {"data": {"id": "tables/users", "label": "Users", "type": "BigQuery Table", "description": "This table contains information about users registered on Stack Overflow, including detailed profile information, activity metrics, and network-wide identifiers.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/users", "tags": ["Stack Overflow", "users", "profiles", "community", "schema", "data dump"], "color": "#3b82f6", "size": 43}}, {"data": {"id": "tables/votes", "label": "Votes", "type": "BigQuery Table", "description": "This table contains information about votes cast on posts within the Stack Overflow platform, including the vote type, associated post and user, and bounty details.", "resource": "https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/votes", "tags": ["votes", "posts", "stackoverflow", "community", "engagement", "schema", "data dump"], "color": "#3b82f6", "size": 43}}], "edges": [{"data": {"id": "datasets/stackoverflow__tables/badges", "source": "datasets/stackoverflow", "target": "tables/badges"}}, {"data": {"id": "datasets/stackoverflow__tables/comments", "source": "datasets/stackoverflow", "target": "tables/comments"}}, {"data": {"id": "datasets/stackoverflow__tables/post_history", "source": "datasets/stackoverflow", "target": "tables/post_history"}}, {"data": {"id": "datasets/stackoverflow__tables/posts_answers", "source": "datasets/stackoverflow", "target": "tables/posts_answers"}}, {"data": {"id": "datasets/stackoverflow__tables/posts_questions", "source": "datasets/stackoverflow", "target": "tables/posts_questions"}}, {"data": {"id": "datasets/stackoverflow__tables/tags", "source": "datasets/stackoverflow", "target": "tables/tags"}}, {"data": {"id": "datasets/stackoverflow__tables/users", "source": "datasets/stackoverflow", "target": "tables/users"}}, {"data": {"id": "datasets/stackoverflow__tables/votes", "source": "datasets/stackoverflow", "target": "tables/votes"}}, {"data": {"id": "references/pending_flags__references/flag_types", "source": "references/pending_flags", "target": "references/flag_types"}}, {"data": {"id": "references/pending_flags__references/close_reason_types", "source": "references/pending_flags", "target": "references/close_reason_types"}}, {"data": {"id": "references/pending_flags__references/close_as_off_topic_reason_types", "source": "references/pending_flags", "target": "references/close_as_off_topic_reason_types"}}, {"data": {"id": "references/post_feedback__references/vote_types", "source": "references/post_feedback", "target": "references/vote_types"}}, {"data": {"id": "references/post_notice_type_definitions__references/post_notice_class_ids", "source": "references/post_notice_type_definitions", "target": "references/post_notice_class_ids"}}, {"data": {"id": "references/post_notice_type_definitions__references/post_notice_duration_ids", "source": "references/post_notice_type_definitions", "target": "references/post_notice_duration_ids"}}, {"data": {"id": "references/post_notices__references/post_notice_types", "source": "references/post_notices", "target": "references/post_notice_types"}}, {"data": {"id": "references/review_rejection_reasons__references/post_type_ids", "source": "references/review_rejection_reasons", "target": "references/post_type_ids"}}, {"data": {"id": "references/review_task_results__references/review_task_result_types", "source": "references/review_task_results", "target": "references/review_task_result_types"}}, {"data": {"id": "references/review_task_results__references/review_rejection_reasons", "source": "references/review_task_results", "target": "references/review_rejection_reasons"}}, {"data": {"id": "references/review_tasks__references/review_task_types", "source": "references/review_tasks", "target": "references/review_task_types"}}, {"data": {"id": "references/review_tasks__references/review_task_states", "source": "references/review_tasks", "target": "references/review_task_states"}}, {"data": {"id": "references/review_tasks__references/review_task_results", "source": "references/review_tasks", "target": "references/review_task_results"}}, {"data": {"id": "references/suggested_edit_votes__references/suggested_edits", "source": "references/suggested_edit_votes", "target": "references/suggested_edits"}}, {"data": {"id": "references/suggested_edit_votes__references/vote_types", "source": "references/suggested_edit_votes", "target": "references/vote_types"}}, {"data": {"id": "tables/badges__tables/users", "source": "tables/badges", "target": "tables/users"}}, {"data": {"id": "tables/badges__references/badge_classes", "source": "tables/badges", "target": "references/badge_classes"}}, {"data": {"id": "tables/comments__datasets/stackoverflow", "source": "tables/comments", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/comments__tables/posts_questions", "source": "tables/comments", "target": "tables/posts_questions"}}, {"data": {"id": "tables/comments__tables/users", "source": "tables/comments", "target": "tables/users"}}, {"data": {"id": "tables/post_history__datasets/stackoverflow", "source": "tables/post_history", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/post_history__references/post_history_type_ids", "source": "tables/post_history", "target": "references/post_history_type_ids"}}, {"data": {"id": "tables/post_history__references/close_reason_types", "source": "tables/post_history", "target": "references/close_reason_types"}}, {"data": {"id": "tables/post_links__datasets/stackoverflow", "source": "tables/post_links", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/post_links__references/link_types", "source": "tables/post_links", "target": "references/link_types"}}, {"data": {"id": "tables/posts_answers__datasets/stackoverflow", "source": "tables/posts_answers", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/posts_answers__tables/posts_questions", "source": "tables/posts_answers", "target": "tables/posts_questions"}}, {"data": {"id": "tables/posts_answers__references/post_type_ids", "source": "tables/posts_answers", "target": "references/post_type_ids"}}, {"data": {"id": "tables/posts_moderator_nomination__tables/users", "source": "tables/posts_moderator_nomination", "target": "tables/users"}}, {"data": {"id": "tables/posts_orphaned_tag_wiki__datasets/stackoverflow", "source": "tables/posts_orphaned_tag_wiki", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/posts_questions__datasets/stackoverflow", "source": "tables/posts_questions", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/posts_questions__tables/posts_answers", "source": "tables/posts_questions", "target": "tables/posts_answers"}}, {"data": {"id": "tables/posts_questions__tables/users", "source": "tables/posts_questions", "target": "tables/users"}}, {"data": {"id": "tables/posts_questions__references/post_type_ids", "source": "tables/posts_questions", "target": "references/post_type_ids"}}, {"data": {"id": "tables/posts_tag_wiki__tables/tags", "source": "tables/posts_tag_wiki", "target": "tables/tags"}}, {"data": {"id": "tables/posts_tag_wiki__datasets/stackoverflow", "source": "tables/posts_tag_wiki", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/stackoverflow_posts__datasets/stackoverflow", "source": "tables/stackoverflow_posts", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/stackoverflow_posts__tables/posts_questions", "source": "tables/stackoverflow_posts", "target": "tables/posts_questions"}}, {"data": {"id": "tables/stackoverflow_posts__tables/posts_answers", "source": "tables/stackoverflow_posts", "target": "tables/posts_answers"}}, {"data": {"id": "tables/tags__datasets/stackoverflow", "source": "tables/tags", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/users__datasets/stackoverflow", "source": "tables/users", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/votes__datasets/stackoverflow", "source": "tables/votes", "target": "datasets/stackoverflow"}}, {"data": {"id": "tables/votes__references/vote_types", "source": "tables/votes", "target": "references/vote_types"}}], "bodies": {"datasets/stackoverflow": "The Stack Overflow public dataset in BigQuery provides a comprehensive collection of data from the popular question and answer platform. This dataset is a valuable resource for researchers and developers interested in analyzing trends in programming questions, user interactions, and the evolution of technology discussions. It encompasses various aspects of the platform, including user profiles, questions, answers, comments, votes, badges awarded, and associated tags.\n\nThis dataset was last updated on 2022-11-25, and it is important to note that it is no longer actively maintained. Despite this, it remains a rich historical archive of Stack Overflow activity. The dataset is hosted in the `US` multi-region.\n\n# Schema\n\nAs a BigQuery Dataset, `stackoverflow` serves as a container for numerous tables rather than having a schema itself. The tables within this dataset collectively represent the structure of Stack Overflow data. Key tables include:\n\n*   [badges](/tables/badges.md): Information about badges awarded to users.\n*   [comments](/tables/comments.md): User comments on posts.\n*   [post_history](/tables/post_history.md): Historical revisions and events for posts.\n*   [posts_answers](/tables/posts_answers.md): Detailed information about answers.\n*   [posts_questions](/tables/posts_questions.md): Detailed information about questions.\n*   [tags](/tables/tags.md): Information about tags used to categorize posts.\n*   [users](/tables/users.md): User profiles and statistics.\n*   [votes](/tables/votes.md): Records of upvotes and downvotes on posts and comments.\n\n# Common query patterns\n\nList all tables within the `stackoverflow` dataset:\n\n```sql\nSELECT table_id\nFROM `bigquery-public-data.stackoverflow.INFORMATION_SCHEMA.TABLES`\nWHERE table_schema = 'stackoverflow';\n```\n\nExample of querying the `posts_questions` table:\n\n```sql\nSELECT\n  id,\n  title,\n  view_count,\n  creation_date\nFROM\n  `bigquery-public-data.stackoverflow.posts_questions`\nWHERE\n  creation_date BETWEEN '2021-01-01' AND '2021-01-31'\nORDER BY\n  view_count DESC\nLIMIT 5;\n```\n\n# Citations\n\n[1] [Stack Overflow Public Dataset](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow)\n[2] [Stack Overflow Official Website](https://stackoverflow.com/)", "references/badge_classes": "# Badge Classes\n\nThis document defines the enumerated classes for badges awarded on Stack Exchange sites.\n\n- `1`: Gold\n- `2`: Silver\n- `3`: Bronze\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/close_as_off_topic_reason_types": "This table defines the enumerated reasons and associated guidance for why a post might be closed as off-topic.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the reason type.\n- `IsUniversal` (BOOLEAN) - Indicates if the reason is universally applicable.\n- `InputTitle` (STRING) - The title shown for input.\n- `MarkdownInputGuidance` (STRING) - Markdown guidance shown while flagging/voting.\n- `MarkdownPostOwnerGuidance` (STRING) - Markdown guidance shown to the original poster when closed.\n- `MarkdownPublicGuidance` (STRING) - Markdown guidance shown to privileged users when closed.\n- `MarkdownConcensusDescription` (STRING) - Markdown description shown above the public or post owner guidance. Nullable.\n- `CreationDate` (TIMESTAMP) - The date and time the reason type was created.\n- `CreationModeratorId` (INTEGER) - The ID of the moderator who created the reason type. Links to the `users` table.\n- `ApprovalDate` (TIMESTAMP) - The date and time the reason type was approved.\n- `ApprovalModeratorId` (INTEGER) - The ID of the moderator who approved the reason type.\n- `DeactivationDate` (TIMESTAMP) - The date and time the reason type was deactivated.\n- `DeactivationModeratorId` (INTEGER) - The ID of the moderator who deactivated the reason type. Links to the `users` table.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/close_reason_types": "This document defines the enumerated types for reasons why a post might be closed on Stack Exchange sites, as found in the `PendingFlags` and `PostHistory` tables.\n\n## Old Close Reasons\n- `1`: Exact Duplicate\n- `2`: Off-topic\n- `3`: Subjective and argumentative\n- `4`: Not a real question\n- `7`: Too localized\n- `10`: General reference\n- `20`: Noise or pointless (Meta sites only)\n\n## Current Close Reasons\n- `101`: Duplicate\n- `102`: Off-topic\n- `103`: Unclear what you\\'re asking\n- `104`: Too broad\n- `105`: Primarily opinion-based\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/creative_commons_by_sa_4_0_license": "All user content contributed to the Stack Exchange network is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0). While this license is permissive, it requires attribution.\n\n## Attribution Requirements for Stack Exchange Content:\n1.  Visually display or otherwise indicate the source of the content as coming from the Stack Exchange Network. This requirement is satisfied with a discreet text blurb, or some other unobtrusive but clear visual indication.\n2.  Ensure that any Internet use of the content includes a hyperlink directly to the original question on the source site on the Network (e.g., `http://stackoverflow.com/questions/12345`).\n3.  Visually display or otherwise clearly indicate the author names for every question and answer used.\n4.  Ensure that any Internet use of the content includes a hyperlink for each author name directly back to his or her user profile page on the source site on the Network (e.g., `http://stackoverflow.com/users/12345/username`), directly to the Stack Exchange domain, in standard HTML (i.e. not through a Tinyurl or other such indirect hyperlink, form of obfuscation or redirection), without any \u201cnofollow\u201d command or any other such means of avoiding detection by search engines, and visible even with JavaScript disabled.\n\nFor more information, see the [Stack Exchange Terms of Service](https://stackexchange.com/legal/terms-of-service).\n\n# Citations\n- [Stack Exchange Data Dump : Internet Archive](https://archive.org/details/stackexchange)\n- [Creative Commons Attribution-ShareAlike 4.0 International License](https://creativecommons.org/licenses/by-sa/4.0/)\n- [Stack Exchange Terms of Service](https://stackexchange.com/legal/terms-of-service)", "references/flag_types": "This document defines the enumerated types for flags related to post moderation on Stack Exchange sites, as found in the `PendingFlags` table.\n\n- `13`: Canned flag for closure\n- `14`: Vote to close\n- `15`: Vote to reopen\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/link_types": "This document defines the enumerated types for links between posts, as found in the `PostLinks` table.\n\n- `1`: Linked (`PostId` contains a link to `RelatedPostId`)\n- `3`: Duplicate (`PostId` is a duplicate of `RelatedPostId`)\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/pending_flags": "This table contains information about pending flags and votes related to post moderation, despite its name focusing on \"PendingFlags\". It includes close-related flags and votes.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the pending flag.\n- `FlagTypeId` (INTEGER) - The type of flag. See the [Flag Types reference](/references/flag_types.md) for possible values:\n    - `13`: canned flag for closure\n    - `14`: vote to close\n    - `15`: vote to reopen\n- `PostId` (INTEGER) - The ID of the post the flag is associated with. Links to the `posts` tables (`posts_questions` or `posts_answers`).\n- `CreationDate` (TIMESTAMP) - The date and time the flag was created.\n- `CloseReasonTypeId` (INTEGER) - The type of close reason, if applicable. See the [Close Reason Types reference](/references/close_reason_types.md) for possible values.\n- `CloseAsOffTopicReasonTypeId` (INTEGER) - The specific off-topic reason ID, if `CloseReasonTypeId = 102` (off-topic). Links to the [Close As Off-Topic Reason Types reference](/references/close_as_off_topic_reason_types.md).\n- `DuplicateOfQuestionId` (INTEGER) - The ID of the duplicate question, if `CloseReasonTypeId` is `1` or `101` (old or current duplicate). Links to the `posts_questions` table.\n- `BelongsOnBaseHostAddress` (STRING) - Indicates the base host address for votes to close and migrate.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_feedback": "This table collects up and down votes from anonymous visitors and/or unregistered users.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the post feedback entry.\n- `PostId` (INTEGER) - The ID of the post (`posts_questions` or `posts_answers`) the feedback is for.\n- `IsAnonymous` (BOOLEAN) - Indicates if the feedback was given anonymously.\n- `VoteTypeId` (INTEGER) - The type of vote. Specifically `2` for UpMod and `3` for DownMod in this table. See the [Vote Types reference](/references/vote_types.md) for a comprehensive list.\n- `CreationDate` (TIMESTAMP) - The date and time the feedback was recorded.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_history_type_ids": "This document defines the enumerated types for various events recorded in the `PostHistory` table, tracking changes and moderation actions on posts across Stack Exchange sites.\n\n- `1`: Initial Title - initial title (questions only)\n- `2`: Initial Body - initial post raw body text\n- `3`: Initial Tags - initial list of tags (questions only)\n- `4`: Edit Title - modified title (questions only)\n- `5`: Edit Body - modified post body (raw markdown)\n- `6`: Edit Tags - modified list of tags (questions only)\n- `7`: Rollback Title - reverted title (questions only)\n- `8`: Rollback Body - reverted body (raw markdown)\n- `9`: Rollback Tags - reverted list of tags (questions only)\n- `10`: Post Closed - post voted to be closed\n- `11`: Post Reopened - post voted to be reopened\n- `12`: Post Deleted - post voted to be removed\n- `13`: Post Undeleted - post voted to be restored\n- `14`: Post Locked - post locked by moderator\n- `15`: Post Unlocked - post unlocked by moderator\n- `16`: Community Owned - post now community owned\n- `17`: Post Migrated - post migrated - now replaced by 35/36 (away/here)\n- `18`: Question Merged - question merged with deleted question\n- `19`: Question Protected - question was protected by a moderator.\n- `20`: Question Unprotected - question was unprotected by a moderator.\n- `21`: Post Disassociated - OwnerUserId removed from post by admin\n- `22`: Question Unmerged - answers/votes restored to previously merged question\n- `24`: Suggested Edit Applied\n- `25`: Post Tweeted\n- `31`: Comment discussion moved to chat\n- `33`: Post notice added - `comment` contains foreign key to PostNotices\n- `34`: Post notice removed - `comment` contains foreign key to PostNotices\n- `35`: Post migrated away - replaces id 17\n- `36`: Post migrated here - replaces id 17\n- `37`: Post merge source\n- `38`: Post merge destination\n- `50`: Bumped by Community User\n- `52`: Question became hot network question (main) / Hot Meta question (meta)\n- `53`: Question removed from hot network/meta questions by a moderator\n- `66`: Created from Ask Wizard\n\n## Older Dumps (Likely no longer present)\n- `23`: Unknown dev related event\n- `26`: Vote nullification by dev (ERM?)\n- `27`: Post unmigrated/hidden moderator migration?\n- `28`: Unknown suggestion event\n- `29`: Unknown moderator event (possibly de-wikification?)\n- `30`: Unknown event (too rare to guess)\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_notice_class_ids": "This document defines the enumerated class identifiers for different types of post notices.\n\n- `0`: DMCA or Redditted\n- `1`: Historical lock\n- `2`: Bounty\n- `4`: Moderator notice (lock)\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_notice_duration_ids": "This document defines the enumerated duration identifiers for post notices.\n\n- `-1`: No duration specified\n- `1`: 7 days (bounty)\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_notice_type_definitions": "This table defines the structure and properties of different types of post notices that can be applied to posts.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the post notice type.\n- `ClassId` (INTEGER) - The class of the notice. See the [Post Notice Class IDs reference](/references/post_notice_class_ids.md) for possible values.\n- `Name` (STRING) - The name of the post notice type.\n- `Body` (STRING) - Contains the default notice text. Nullable.\n- `IsHidden` (BOOLEAN) - Indicates if the notice type is hidden.\n- `Predefined` (BOOLEAN) - Indicates if the notice type is predefined.\n- `PostNoticeDurationId` (INTEGER) - The duration identifier for the notice. See the [Post Notice Duration IDs reference](/references/post_notice_duration_ids.md) for possible values.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_notice_types": "This document defines the enumerated types for notices that can be applied to posts on Stack Exchange sites, as found in the `PostNotices` table.\n\n- `1`: Citation needed\n- `2`: Current event\n- `3`: Insufficient explanation\n- `10`: Current answers are outdated\n- `11`: Draw attention\n- `12`: Improve details\n- `13`: Authoritative reference needed\n- `14`: Canonical answer required\n- `15`: Reward existing answer\n- `20`: Content dispute\n- `21`: Offtopic comments\n- `22`: Historical significance\n- `23`: Wiki Answer\n- `24`: Policy Lock (SO Collectives)\n- `25`: Recommended Answer (SO Collectives)\n- `26`: Posted by Recognized Member/Admin (SO Collectives)\n- `27`: Endorsed Edit (SO Collectives)\n- `28`: Obsolete (SO Collectives)\n- `1000`: Redditted (SO)\n- `9001`: DMCA Takedown\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_notices": "This table stores information about notices applied to posts, such as \"Citation needed\" or \"Duplicate\".\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the post notice.\n- `PostId` (INTEGER) - The ID of the post the notice is applied to. Links to the `posts` tables (`posts_questions` or `posts_answers`).\n- `PostNoticeTypeId` (INTEGER) - The type of post notice. See the [Post Notice Types reference](/references/post_notice_types.md) for possible values.\n- `CreationDate` (TIMESTAMP) - The date and time the notice was created.\n- `DeletionDate` (TIMESTAMP) - The date and time the notice was deleted. Nullable.\n- `ExpiryDate` (TIMESTAMP) - The date and time the notice is set to expire. Nullable.\n- `Body` (STRING) - When present, contains the custom text shown with the notice. Nullable.\n- `OwnerUserId` (INTEGER) - The ID of the user who created the notice. Links to the `users` table. Nullable.\n- `DeletionUserId` (INTEGER) - The ID of the user who deleted the notice. Links to the `users` table. Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/post_type_ids": "# Post Type IDs\n\nThis document defines the enumerated IDs for different types of posts found in the Stack Exchange data dump and SEDE.\n\n- `1`: Question\n- `2`: Answer\n- `3`: Orphaned Tag Wiki\n- `4`: Tag Wiki\n- `5`: Tag Wiki Excerpt\n- `6`: Moderator Nomination\n- `7`: Privilege Wiki\n- `8`: Status Wiki\n- `10`: Deleted Question (not in public data dump)\n- `11`: Section\n- `12`: Article\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/related_questions": "This table provides a mapping between questions and other related questions, based on factors like content similarity or user behavior.\n\n## Schema\n- `PostId` (INTEGER) - The ID of the primary question. Links to the `posts_questions` table.\n- `RelatedPostId` (INTEGER) - The ID of a related question. Links to the `posts_questions` table.\n- `Position` (INTEGER) - The position or ranking of the related question.\n- `Score` (INTEGER) - A score indicating the relevance or strength of the relationship.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_rejection_reasons": "This table defines canned rejection reasons for suggested edits.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the rejection reason.\n- `Name` (STRING) - The name of the rejection reason.\n- `Description` (STRING) - A detailed description of the rejection reason.\n- `PostTypeId` (INTEGER) - The type of post the reason applies to (e.g., `5` for Wiki, `6` for Excerpt). Otherwise, it is null. Links to the [Post Type IDs reference](/references/post_type_ids.md).\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_task_result_types": "This document defines the enumerated types for the outcomes of various review tasks on Stack Exchange sites.\n\n- `1`: Skip\n- `2`: Approve (suggested edits)\n- `3`: Reject (suggested edits)\n- `4`: Delete (low quality)\n- `5`: Edit (first posts, late answers, low quality)\n- `6`: Close (close, low quality)\n- `7`: Looks OK (low quality)\n- `8`: Do Not Close (close)\n- `9`: Recommend Deletion (low quality answer)\n- `10`: Recommend Close (low quality question)\n- `11`: Other Action (first posts), previously \"I\\'m Done\"\n- `12`: Reopen (reopen)\n- `13`: Leave Closed (reopen)\n- `14`: Edit and Reopen (reopen)\n- `15`: Excellent (community evaluation)\n- `16`: Satisfactory (community evaluation)\n- `17`: Needs Improvement (community evaluation)\n- `18`: No Action Needed (first posts, late answers)\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_task_results": "This table stores the outcomes and details of completed review tasks, such as approvals, rejections, or edits.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the review task result.\n- `ReviewTaskId` (INTEGER) - The ID of the review task this result is for. Links to the `ReviewTasks` table.\n- `ReviewTaskResultTypeId` (INTEGER) - The type of result. See the [Review Task Result Types reference](/references/review_task_result_types.md) for possible values.\n- `CreationDate` (TIMESTAMP) - The date when the result was recorded. Time data is purposefully removed to protect user privacy.\n- `RejectionReasonId` (INTEGER) - The ID of the rejection reason, specifically for suggested edits. Links to the [Review Rejection Reasons reference](/references/review_rejection_reasons.md). Nullable.\n- `Comment` (STRING) - A comment associated with the review result. Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_task_states": "This document defines the enumerated states for review tasks on Stack Exchange sites.\n\n- `1`: Active\n- `2`: Completed\n- `3`: Invalidated\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_task_types": "This document defines the enumerated types for various review tasks available on Stack Exchange sites.\n\n- `1`: Suggested Edit\n- `2`: Close Votes\n- `3`: Low Quality Posts\n- `4`: First Post\n- `5`: Late Answer\n- `6`: Reopen Vote\n- `7`: Community Evaluation\n- `8`: Link Validation\n- `9`: Flagged Posts\n- `10`: Triage\n- `11`: Helper\n- `12`: First Questions\n- `13`: First Answers\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/review_tasks": "This table stores information about moderation review tasks on Stack Exchange, such as suggested edits or close votes.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the review task.\n- `ReviewTaskTypeId` (INTEGER) - The type of review task. See the [Review Task Types reference](/references/review_task_types.md) for possible values.\n- `CreationDate` (TIMESTAMP) - The date when the review task was created. Time data is typically removed to protect user privacy.\n- `DeletionDate` (TIMESTAMP) - The date when the review task was deleted. Time data is typically removed to protect user privacy. Nullable.\n- `ReviewTaskStateId` (INTEGER) - The current state of the review task. See the [Review Task States reference](/references/review_task_states.md) for possible values.\n- `PostId` (INTEGER) - The ID of the post associated with the review task. Links to the `posts` tables (`posts_questions` or `posts_answers`). Nullable.\n- `SuggestedEditId` (INTEGER) - The ID of the suggested edit associated with the review task, if applicable. Links to the `SuggestedEdits` table. Nullable.\n- `CompletedByReviewTaskId` (INTEGER) - The ID associated with the `ReviewTaskResult` that stores the outcome of a completed review. Links to the [Review Task Results reference](/references/review_task_results.md). Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_databases": "This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each Stack Exchange site\\'s database within SEDE.\n\n## Schema\n- `database_id` (INTEGER) - Unique identifier for the database.\n- `database_name` (STRING) - The name of the database.\n- `site_name` (STRING) - The short name of the site.\n- `tiny_name` (STRING) - A very short name for the site.\n- `long_name` (STRING) - The full name of the site.\n- `site_type` (STRING) - Type of site, either \\'main_site\\' or \\'meta_site\\'.\n- `site_url` (STRING) - The base URL of the Stack Exchange site.\n- `sede_url` (STRING) - The URL to the site\\'s data in SEDE.\n- `api_site_parameter` (STRING) - The parameter used for the site in the Stack Exchange API.\n- `initialized` (TIMESTAMP) - Datetime when populating that database started.\n- `made_available` (TIMESTAMP) - Datetime when the database came online and was ready for queries.\n- `processing_time` (STRING) - The duration between `initialized` and `made_available`, in `hh:mm:ss.fff` format.\n- `questions` (INTEGER) - Total number of questions in this site at the time of the current refresh.\n- `answers` (INTEGER) - Total number of answers in this site at the time of the current refresh.\n- `latest_post` (TIMESTAMP) - The timestamp of the last non-deleted post captured in this refresh.\n- `notes` (STRING) - This will be non-NULL when a database is in transition. Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_databases_markdown": "This internal SEDE table contains markdown content related to database information, likely for display purposes.\n\n## Schema\n- `rn` (INTEGER) - Row number, used for sorting by the \\'long name\\' of the database.\n- `content` (STRING) - Markdown content, potentially including headers, for database descriptions.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_ineachdb": "This is an internal stored procedure in the Stack Exchange Data Explorer (SEDE) used to execute SQL commands across multiple databases (i.e., Stack Exchange sites).\n\n## Parameters\n| Parameter           | Data Type      | Default | Description                                                               |\n|---------------------|----------------|---------|---------------------------------------------------------------------------|\n| `@SQLCommand`       | `nvarchar(4000)` |         | The SQL statement to run.                                                 |\n| `@IncludeMainSites` | `bit`          | `1`     | Set to `1` to include all non-meta sites.                                 |\n| `@IncludeMetaSites` | `bit`          | `1`     | Set to `1` to include all meta sites.                                     |\n| `@IncludeMainMeta`  | `bit`          | `1`     | Set to `1` to include `StackExchange.Meta`.                               |\n| `@CollectResultsForMe` | `bit`          | `1`     | For standard `SELECT` queries, this will attempt to put each database's results into a `#temp` table. |\n| `@ErrorOnSkippedSites` | `bit`          | `0`     | Set this to `1` if you want execution to halt in the event any site is missing due to transition change. |\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_sites": "This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each Stack Exchange site.\n\n## Schema\n- `site_name` (STRING) - The short name of the site.\n- `site_url` (STRING) - The base URL of the Stack Exchange site.\n- `database_name` (STRING) - The name of the corresponding database in SEDE.\n- `long_name` (STRING) - The full name of the site.\n- `site_id` (INTEGER) - Unique identifier for the site.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_tables": "This internal table in the Stack Exchange Data Explorer (SEDE) provides metadata about each table within SEDE databases.\n\n## Schema\n- `database_id` (INTEGER) - The ID of the database, originating from `sys.databases`.\n- `database_name` (STRING) - The name of the database, originating from `sys.databases`.\n- `table_name` (STRING) - The name of the table, originating from `sys.tables`.\n- `latest_date` (TIMESTAMP) - The latest timestamp (e.g., `CreationDate`) for tables that have such a column. Nullable.\n- `row_count` (INTEGER) - The number of rows in the table, according to `sys.partitions`. For `Posts`, this includes `PostsWithDeleted`; for non-deleted `Posts`, refer to `answers + questions` from `dbo.sede_databases`.\n- `initialized` (TIMESTAMP) - Timestamp of initial table creation.\n- `made_available` (TIMESTAMP) - Timestamp of the final operation against the table.\n- `processing_time` (STRING) - Duration of `initialized` -> `made_available`, in `{hh:mm:ss.fff}` format.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/sede_users": "This internal table in the Stack Exchange Data Explorer (SEDE) provides user-specific metadata across sites.\n\n## Schema\n- `account_id` (INTEGER) - The global Stack Exchange Network account ID.\n- `site_id` (INTEGER) - The ID of the site the user is associated with. Links to `references/sede_sites.md`.\n- `user_id` (INTEGER) - The local user ID on that specific site. Links to the `users` table.\n- `reputation` (INTEGER) - The user\\'s reputation on that specific site.\n- `question_count` (INTEGER) - The number of questions asked by the user on that specific site.\n- `answer_count` (INTEGER) - The number of answers provided by the user on that specific site.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/suggested_edit_votes": "This table records votes cast on suggested edits, indicating approval or rejection.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the suggested edit vote.\n- `SuggestedEditId` (INTEGER) - The ID of the suggested edit that was voted on. Links to the [Suggested Edits reference](/references/suggested_edits.md).\n- `UserId` (INTEGER) - The ID of the user who cast the vote. Links to the `users` table.\n- `VoteTypeId` (INTEGER) - The type of vote. Specifically `2` for Approve (UpMod) and `3` for Reject (DownMod). See the [Vote Types reference](/references/vote_types.md) for a comprehensive list.\n- `CreationDate` (TIMESTAMP) - The date and time the vote was cast.\n- `TargetUserId` (INTEGER) - The ID of the target user for the vote (e.g., the user whose reputation changed). Links to the `users` table. Nullable.\n- `TargetRepChange` (INTEGER) - The change in reputation for the target user as a result of this vote. Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/suggested_edits": "This table stores information about suggested edits to posts that are awaiting review. If both `ApprovalDate` and `RejectionDate` are null, then the edit is still in review.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the suggested edit.\n- `PostId` (INTEGER) - The ID of the post to which the suggested edit applies. Links to the `posts` tables (`posts_questions` or `posts_answers`).\n- `CreationDate` (TIMESTAMP) - The date and time the suggested edit was created.\n- `ApprovalDate` (TIMESTAMP) - The date and time the suggested edit was approved. Null if not yet approved.\n- `RejectionDate` (TIMESTAMP) - The date and time the suggested edit was rejected. Null if not yet rejected.\n- `OwnerUserId` (INTEGER) - The ID of the user who suggested the edit. Links to the `users` table.\n- `Comment` (STRING) - A comment provided with the suggested edit. Nullable.\n- `Text` (STRING) - The proposed new body text for the post. Nullable.\n- `Title` (STRING) - The proposed new title for the post. Nullable.\n- `Tags` (STRING) - The proposed new tags for the post. Nullable.\n- `RevisionGUID` (STRING) - A GUID for the revision.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/tag_synonyms": "This table stores relationships between synonymous tags, allowing for automatic renaming or suggestion of alternative tags.\n\n## Schema\n- `Id` (INTEGER) - Unique identifier for the tag synonym.\n- `SourceTagName` (STRING) - The name of the tag that is considered a synonym. Nullable. Links to the `tags` table\\'s `tag_name` field.\n- `TargetTagName` (STRING) - The name of the canonical tag to which `SourceTagName` refers. Nullable. Links to the `tags` table\\'s `tag_name` field.\n- `CreationDate` (TIMESTAMP) - The date and time the tag synonym was created.\n- `OwnerUserId` (INTEGER) - The ID of the user who created the tag synonym. Nullable. Links to the `users` table.\n- `AutoRenameCount` (INTEGER) - The number of times this synonym has resulted in an automatic tag rename.\n- `LastAutoRename` (TIMESTAMP) - The date and time of the last automatic rename using this synonym. Nullable.\n- `Score` (INTEGER) - A score indicating the quality or consensus of the synonym.\n- `ApprovedByUserId` (INTEGER) - The ID of the user who approved the tag synonym. Nullable. Links to the `users` table.\n- `ApprovalDate` (TIMESTAMP) - The date and time the tag synonym was approved. Nullable.\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "references/vote_types": "This document defines the enumerated types for various votes that can occur on posts and other entities across Stack Exchange sites.\n\n- `-1`: InformModerator\n- `0`: UndoMod\n- `1`: AcceptedByOriginator\n- `2`: UpMod (Upvote)\n- `3`: DownMod (Downvote)\n- `4`: Offensive\n- `5`: Favorite (Bookmark) - feature removed after October 2022, replaced by Saves.\n- `6`: Close - Close votes are only stored in the `PostHistory` table after 2013-06-25.\n- `7`: Reopen\n- `8`: BountyStart\n- `9`: BountyClose\n- `10`: Deletion\n- `11`: Undeletion\n- `12`: Spam\n- `15`: ModeratorReview - a moderator looking at a flagged post.\n- `16`: ApproveEditSuggestion\n- `17`: Reaction1 (Teams: celebrate)\n- `18`: Helpful\n- `19`: ThankYou\n- `20`: WellWritten\n- `21`: Follow\n- `22`: Reaction2 (Teams: smile)\n- `23`: Reaction3 (Teams: mind blown)\n- `24`: Reaction4 (Teams: clap)\n- `25`: Reaction5 (Teams: heart)\n- `26`: Reaction6 (Teams: fire)\n- `27`: Reaction7 (Teams: trophy)\n- `28`: Reaction8 (Teams: wave)\n- `29`: Outdated\n- `30`: NotOutdated\n- `31`: PreVote\n- `32`: CollectiveDiscussionUpvote\n- `33`: CollectiveDiscussionDownvote (no longer used)\n- `35`: privateAiAnswerCorrect\n- `36`: privateAiAnswerIncorrect\n- `37`: privateAiAnswerPartiallyCorrect\n\n# Citations\n- [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/badges": "The `badges` table records all badges awarded to users on the Stack Overflow Q&A platform. Each row represents a single badge awarded, detailing its unique identifier, name, the date it was awarded, the ID of the user who received it, its class (e.g., gold, silver, bronze), and whether it is a tag-based badge. The data spans from October 26, 2016, up to November 24, 2022. This table can be joined with the [users](/tables/users.md) table on `user_id` to get more information about the badge recipients.\n\n# Schema\n\n- `id` (INTEGER) - Unique identifier for the badge award.\n- `name` (STRING) - Name of the badge.\n- `date` (TIMESTAMP) - Timestamp when the badge was awarded (e.g., `2008-09-15T08:55:03.923`).\n- `user_id` (INTEGER) - Identifier of the user who received the badge.\n- `class` (INTEGER) - The class of the badge. See the [Badge Classes reference](/references/badge_classes.md) for possible values.\n- `tag_based` (BOOLEAN) - `True` if the badge is for a tag, otherwise it is a named badge.\n\n# Common query patterns\n\n```sql\nSELECT\n    name,\n    COUNT(*) AS badge_count\n  FROM\n    `bigquery-public-data.stackoverflow.badges`\n  GROUP BY\n    name\n  ORDER BY\n    badge_count DESC\n  LIMIT 5;\n```\n\n```sql\nSELECT\n    b.name,\n    b.date\n  FROM\n    `bigquery-public-data.stackoverflow.badges` AS b\n    JOIN `bigquery-public-data.stackoverflow.users` AS u ON b.user_id = u.id\n  WHERE\n    u.display_name = \\'Jon Skeet\\'\n  ORDER BY\n    b.date DESC;\n```\n\n```sql\nSELECT\n    DATE(date) AS award_date,\n    COUNT(*) AS badges_awarded\n  FROM\n    `bigquery-public-data.stackoverflow.badges`\n  GROUP BY\n    award_date\n  ORDER BY\n    award_date DESC\n  LIMIT 10;\n```\n\n# Citations\n\n[1] [BigQuery Public Data: stackoverflow.badges](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/badges)\n[2] [Stack Overflow](https://stackoverflow.com/)\n[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/comments": "The `comments` table in the [Stack Overflow dataset](/datasets/stackoverflow.md) captures all user-generated comments on various posts across the platform. Each row in this table represents a unique comment, providing details such as the comment\\'s text, its creation timestamp, and a score reflecting its community reception. Comments are linked to the specific [posts](/tables/posts_questions.md) (questions or answers) they belong to via `post_id` and to the [users](/tables/users.md) who authored them via `user_id`. This table is crucial for analyzing user engagement, content quality feedback, and conversational threads on the platform.\n\n# Schema\n- `id` (INTEGER) - Unique identifier for the comment.\n- `post_id` (INTEGER) - The ID of the post (question or answer) to which the comment belongs.\n- `score` (INTEGER) - The score of the comment, reflecting upvotes and downvotes.\n- `text` (STRING) - The content of the comment (Comment body).\n- `creation_date` (TIMESTAMP) - The date and time the comment was posted.\n- `user_display_name` (STRING) - The display name of the user who made the comment. Nullable.\n- `user_id` (INTEGER) - The ID of the user who made the comment. Optional, absent if user has been deleted.\n- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.\n\n# Common query patterns\n```sql\n-- Retrieve the 10 most recent comments\nSELECT\n  id,\n  text,\n  creation_date,\n  user_display_name\nFROM\n  `bigquery-public-data.stackoverflow.comments`\nORDER BY\n  creation_date DESC\nLIMIT 10;\n```\n\n```sql\n-- Count comments per user\nSELECT\n  user_display_name,\n  COUNT(id) AS total_comments\nFROM\n  `bigquery-public-data.stackoverflow.comments`\nWHERE\n  user_display_name IS NOT NULL\nGROUP BY\n  user_display_name\nORDER BY\n  total_comments DESC\nLIMIT 10;\n```\n\n```sql\n-- Find comments with a high score for a specific post\nSELECT\n  id,\n  text,\n  score,\n  creation_date\nFROM\n  `bigquery-public-data.stackoverflow.comments`\nWHERE\n  post_id = 45651 AND score > 5\nORDER BY\n  score DESC;\n```\n\n# Citations\n[1] [BigQuery Public Dataset: Stack Overflow Comments](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/comments)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/post_history": "This table, part of the [Stack Overflow dataset](/datasets/stackoverflow.md), records every historical event related to posts, such as edits, rollbacks, and changes in post type. Each row represents a single historical event for a particular post, identified by `post_id`, with the event\\'s timestamp recorded in `creation_date`. This allows for a comprehensive audit trail of how content on Stack Overflow evolves over time.\n\n### Schema\n\n- `id` (INTEGER) - Unique identifier for the post history entry.\n- `post_history_type_id` (INTEGER) - An integer representing the type of history event. See the [Post History Type IDs reference](/references/post_history_type_ids.md) for possible values.\n- `post_id` (INTEGER) - The ID of the post to which this history event belongs. Links to `posts_questions` or `posts_answers` tables.\n- `revision_guid` (STRING) - A GUID that groups multiple history records caused by a single action.\n- `creation_date` (TIMESTAMP) - The timestamp when the history event occurred (e.g., `2009-03-05T22:28:34.823`).\n- `user_id` (INTEGER) - The ID of the user who performed the action, if applicable. Links to the `users` table.\n- `user_display_name` (STRING) - Populated if a user has been removed and is no longer referenced by `user_id`; also happens to the author of a migrated post. Nullable.\n- `comment` (STRING) - A brief comment describing the history event. This field can also contain specific IDs based on `PostHistoryTypeId`:\n    - If `PostHistoryTypeId = 10`, it contains the `CloseReasonId`. See the [Close Reason Types reference](/references/close_reason_types.md).\n    - If `PostHistoryTypeId` is `33` or `34`, it contains the `PostNoticeId`.\n- `text` (STRING) - A raw version of the new value for a given revision. This field can also contain specific JSON encoded strings based on `PostHistoryTypeId`:\n    - If `PostHistoryTypeId` is `10, 11, 12, 13, 14, 15, 19, 20, 35`, it contains a JSON encoded string with all users who have voted for that `PostHistoryTypeId`.\n    - If it is a duplicate close vote, the JSON string contains an array of original questions as `OriginalQuestionIds`.\n    - If `PostHistoryTypeId = 17`, it contains migration details (`from <url>` or `to <url>`).\n- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.\n\n### Common query patterns\n\n```sql\nSELECT\n  creation_date,\n  post_history_type_id,\n  user_id,\n  comment,\n  text\nFROM\n  `bigquery-public-data.stackoverflow.post_history`\nWHERE\n  post_id = 456789 -- Example post ID\nORDER BY\n  creation_date DESC\nLIMIT 10;\n```\n\n```sql\nSELECT\n  post_history_type_id,\n  COUNT(*) AS event_count\nFROM\n  `bigquery-public-data.stackoverflow.post_history`\nGROUP BY\n  post_history_type_id\nORDER BY\n  event_count DESC;\n```\n\n```sql\nSELECT\n  t2.display_name,\n  COUNT(t1.id) AS edit_count\nFROM\n  `bigquery-public-data.stackoverflow.post_history` AS t1\nINNER JOIN\n  `bigquery-public-data.stackoverflow.users` AS t2\nON\n  t1.user_id = t2.id\nWHERE\n  t1.post_history_type_id = 2 -- Assuming \\'2\\' means \\'Post Edited\\'\nGROUP BY\n  t2.display_name\nORDER BY\n  edit_count DESC\nLIMIT 5;\n```\n\n### Citations\n\n[1] [BigQuery Public Dataset: Stack Overflow Post History](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_history)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/post_links": "The `post_links` table in the [stackoverflow](/datasets/stackoverflow.md) dataset records relationships between posts on the Stack Overflow platform. Each row represents a single link between two posts, identifying the `post_id` and `related_post_id`. This table can be used to understand how discussions evolve, identify duplicate questions, or find related content across the platform. The `link_type_id` categorizes the nature of the relationship, such as \"Linked\" or \"Duplicate\".\n\n# Schema\n\n*   `id` (INTEGER) - Unique identifier for the post link (primary key).\n*   `creation_date` (TIMESTAMP) - The timestamp when the link was created.\n*   `link_type_id` (INTEGER) - The type of link. See the [Link Types reference](/references/link_types.md) for possible values.\n*   `post_id` (INTEGER) - The ID of the source post. Links to the `id` in the `posts_questions` or `posts_answers` tables.\n*   `related_post_id` (INTEGER) - The ID of the target/related post. Links to the `id` in the `posts_questions` or `posts_answers` tables.\n\n# Common query patterns\n\n```sql\nSELECT\n  pl.post_id,\n  pl.related_post_id,\n  pl.creation_date,\n  pl.link_type_id\nFROM\n  `bigquery-public-data.stackoverflow.post_links` AS pl\nWHERE\n  pl.post_id = 12345\nLIMIT 100;\n```\n\n```sql\nSELECT\n  link_type_id,\n  COUNT(*) AS link_count\nFROM\n  `bigquery-public-data.stackoverflow.post_links`\nGROUP BY\n  link_type_id\nORDER BY\n  link_count DESC;\n```\n\n# Citations\n\n[1] [BigQuery Public Dataset: Stack Overflow Post Links](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/post_links)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/posts_answers": "The `posts_answers` table in the [stackoverflow](/datasets/stackoverflow.md) dataset contains all posts identified as answers to questions on the Stack Overflow platform. Each row in this table represents a single answer, providing details such as the answer\\'s body, its score, creation date, and the ID of the parent question it answers. This table can be joined with other tables like [posts_questions](/tables/posts_questions.md) on `parent_id` to link answers to their respective questions. It\\'s a valuable resource for analyzing answer quality, user contributions, and engagement patterns within the Stack Overflow community.\n\n# Schema\n- `id` (INTEGER) - Unique identifier for the answer.\n- `title` (STRING) - The title of the post. This is typically NULL for answers.\n- `body` (STRING) - The body of the post, as rendered HTML, not Markdown.\n- `accepted_answer_id` (INTEGER) - The ID of the answer that was accepted for a question. This field is only applicable to questions, so it is typically NULL for answers.\n- `answer_count` (INTEGER) - The number of answers. This field is only applicable to questions, so it is typically NULL for answers.\n- `closed_date` (TIMESTAMP) - The date and time the post was closed; present only if the post is closed. This is typically NULL for answers.\n- `comment_count` (INTEGER) - The number of comments.\n- `community_owned_date` (TIMESTAMP) - The date and time the post became community wiki\\'d.\n- `creation_date` (TIMESTAMP) - The date and time the answer was created.\n- `favorite_count` (INTEGER) - The number of times the post has been favorited. Nullable. This is typically NULL for answers.\n- `last_activity_date` (TIMESTAMP) - The datetime of the post\\'s most recent activity.\n- `last_edit_date` (TIMESTAMP) - The date and time of the most recent edit to the answer.\n- `last_editor_display_name` (STRING) - The display name of the last editor. Nullable.\n- `last_editor_user_id` (INTEGER) - The ID of the last editor user. Nullable.\n- `owner_display_name` (STRING) - The display name of the answer\\'s owner. Nullable.\n- `owner_user_id` (INTEGER) - The ID of the answer\\'s owner. Present only if user has not been deleted.\n- `parent_id` (INTEGER) - The ID of the question this answer belongs to. Present only if `PostTypeId = 2`.\n- `post_type_id` (INTEGER) - The type of post. For answers, this is `2`. See the [Post Type IDs reference](/references/post_type_ids.md) for possible values.\n- `score` (INTEGER) - The score of the answer; generally non-zero for Questions, Answers, and Moderator Nominations.\n- `tags` (STRING) - Tags associated with the post. This is typically NULL for answers, as tags are associated with the parent question.\n- `view_count` (INTEGER) - The number of times the post was viewed. This field is only applicable to questions, so it is typically NULL for answers.\n- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.\n\n# Common query patterns\n1. Get the 10 most highly-scored answers in a specific date range:\n```sql\nSELECT\n  id,\n  score,\n  creation_date,\n  body\nFROM\n  `bigquery-public-data.stackoverflow.posts_answers`\nWHERE\n  creation_date BETWEEN \\'2023-01-01\\' AND \\'2023-01-31\\'\nORDER BY\n  score DESC\nLIMIT 10\n```\n2. Find the total number of answers by a specific user:\n```sql\nSELECT\n  owner_user_id,\n  COUNT(id) AS total_answers\nFROM\n  `bigquery-public-data.stackoverflow.posts_answers`\nWHERE\n  owner_user_id = 12345 -- Replace with a valid user ID\nGROUP BY\n  owner_user_id\n```\n3. Join answers with their corresponding questions:\n```sql\nSELECT\n  q.title AS question_title,\n  a.body AS answer_body,\n  a.score AS answer_score\nFROM\n  `bigquery-public-data.stackoverflow.posts_answers` AS a\nJOIN\n  `bigquery-public-data.stackoverflow.posts_questions` AS q\nON\n  a.parent_id = q.id\nWHERE\n  q.id = 5000 -- Example question ID\n```\n\n# Citations\n[1] [Stack Overflow Posts Answers Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_answers)\n[2] [Stack Overflow Public Data Explorer](https://data.stackexchange.com/)\n[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/posts_moderator_nomination": "This table contains posts submitted by users nominating themselves or others to become moderators on the Stack Overflow platform. Each row represents a single moderator nomination post, detailing the candidate's qualifications or reasons for nomination.\n\nThe `creation_date` field indicates when the nomination post was created, providing a historical record of moderator elections and community engagement.\n\n# Schema\n\n- `id`: Unique identifier for the post.\n- `title`: Title of the nomination post (often null for this post type).\n- `body`: The content of the nomination post, describing the candidate's qualifications.\n- `accepted_answer_id`: (STRING) ID of the accepted answer, if applicable.\n- `answer_count`: (STRING) Number of answers to the post.\n- `comment_count`: Number of comments on the post.\n- `community_owned_date`: Timestamp when the post became community-owned.\n- `creation_date`: Timestamp when the post was created.\n- `favorite_count`: (STRING) Number of times the post has been favorited.\n- `last_activity_date`: Timestamp of the last activity on the post.\n- `last_edit_date`: Timestamp of the last edit to the post.\n- `last_editor_display_name`: Display name of the last editor.\n- `last_editor_user_id`: User ID of the last editor.\n- `owner_display_name`: Display name of the post owner.\n- `owner_user_id`: User ID of the post owner. This links to the [/users](/tables/users.md) table.\n- `parent_id`: (STRING) ID of the parent post, if this is a reply.\n- `post_type_id`: Type of the post (e.g., 6 for moderator nomination).\n- `score`: The score of the post.\n- `tags`: (STRING) Tags associated with the post.\n- `view_count`: (STRING) Number of views the post has received.\n\n# Common query patterns\n\n```sql\nSELECT\n    id,\n    creation_date,\n    owner_user_id,\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_moderator_nomination`\n  WHERE\n    creation_date BETWEEN '2021-01-01' AND '2021-12-31'\n  ORDER BY\n    creation_date DESC\n  LIMIT 100\n```\n\n```sql\nSELECT\n    p.id,\n    p.creation_date,\n    u.display_name AS nominator_name,\n    p.body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_moderator_nomination` AS p\n  JOIN\n    `bigquery-public-data.stackoverflow.users` AS u\n    ON p.owner_user_id = u.id\n  WHERE\n    p.creation_date >= '2022-01-01'\n  LIMIT 10\n```\n\n# Citations\n\n[1] [BigQuery Public Data: Stack Overflow Posts Moderator Nomination](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_moderator_nomination)", "tables/posts_orphaned_tag_wiki": "This table, part of the public [Stack Overflow dataset](/datasets/stackoverflow.md), contains posts that were originally created as Tag Wikis but are no longer linked to an existing tag within the Stack Overflow platform. These 'orphaned' Tag Wiki entries typically describe the purpose and usage of a specific programming language, technology, or concept. Each row represents a single Tag Wiki post, identified by a unique `id`. The `post_type_id` for these entries is typically `3`. The table includes the body of the wiki, creation dates, and metadata about the last editor and activity. It can be used to analyze historical Tag Wiki content that has become disassociated from its original tag.\n\n# Schema\n\n- `id`: Unique identifier for the post.\n- `title`: Title of the post (often null for Tag Wikis).\n- `body`: The main content of the Tag Wiki, often including HTML.\n- `accepted_answer_id`: (Always null for this post type).\n- `answer_count`: (Always null for this post type).\n- `comment_count`: Number of comments on the post.\n- `community_owned_date`: Timestamp when the post became community-owned.\n- `creation_date`: Timestamp when the post was created.\n- `favorite_count`: Number of times the post has been favorited.\n- `last_activity_date`: Timestamp of the last activity on the post.\n- `last_edit_date`: Timestamp of the last edit to the post.\n- `last_editor_display_name`: Display name of the last user to edit the post.\n- `last_editor_user_id`: User ID of the last user to edit the post.\n- `owner_display_name`: Display name of the user who owns the post.\n- `owner_user_id`: User ID of the user who owns the post.\n- `parent_id`: (Always null for this post type).\n- `post_type_id`: Type of the post. For this table, it's typically `3` (Tag Wiki).\n- `score`: The score of the post.\n- `tags`: Tags associated with the post (often null as these are orphaned).\n- `view_count`: Number of times the post has been viewed.\n\n# Common query patterns\n\n```sql\nSELECT\n    id,\n    creation_date,\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`\n  WHERE\n    creation_date < '2015-01-01'\n  LIMIT 100;\n```\n\n```sql\nSELECT\n    id,\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`\n  WHERE\n    id = 4164933;\n```\n\n```sql\nSELECT\n    count(id) AS total_orphaned_tag_wikis\n  FROM\n    `bigquery-public-data.stackoverflow.posts_orphaned_tag_wiki`;\n```\n\n# Citations\n\n[1] [BigQuery Public Data: Stack Overflow posts_orphaned_tag_wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_orphaned_tag_wiki)\n[2] [Stack Overflow](https://stackoverflow.com/)", "tables/posts_privilege_wiki": "This BigQuery table, `posts_privilege_wiki`, contains posts that describe various user privileges within the Stack Overflow community. These entries serve as informational guides, explaining what a privilege entails, when and how to use it, and related community guidelines. Examples include details on editing questions and answers, or the process of retagging questions. Each row represents a specific privilege explanation, detailing its purpose and usage, contributing to the self-documentation of the platform's community features.\n\n# Schema\n- `id`: Unique identifier for the post.\n- `title`: The title of the post. For these privilege wiki entries, titles are often null, with the main content in the body.\n- `body`: The full content of the privilege explanation, often including HTML formatting.\n- `accepted_answer_id`: (Always null for these types of posts)\n- `answer_count`: (Always null for these types of posts)\n- `comment_count`: Number of comments on the post.\n- `community_owned_date`: Date when the post became community-owned.\n- `creation_date`: Timestamp when the post was created.\n- `favorite_count`: (Always null for these types of posts)\n- `last_activity_date`: Timestamp of the last activity on the post.\n- `last_edit_date`: Timestamp of the last edit to the post.\n- `last_editor_display_name`: Display name of the last user who edited the post.\n- `last_editor_user_id`: User ID of the last user who edited the post.\n- `owner_display_name`: Display name of the post's owner.\n- `owner_user_id`: User ID of the post's owner. Often -1 for community posts.\n- `parent_id`: (Always null for these types of posts)\n- `post_type_id`: Type of post, typically 8 for wiki entries.\n- `score`: The score (upvotes minus downvotes) of the post.\n- `tags`: (Always null for these types of posts)\n- `view_count`: (Always null for these types of posts)\n\n# Common query patterns\n```sql\nSELECT\n  id,\n  body\nFROM\n  `bigquery-public-data.stackoverflow.posts_privilege_wiki`\nWHERE\n  creation_date >= '2013-01-01'\nLIMIT 100\n```\n```sql\nSELECT\n  id,\n  body,\n  creation_date\nFROM\n  `bigquery-public-data.stackoverflow.posts_privilege_wiki`\nWHERE\n  body LIKE '%edit questions and answers%'\n```\n\n# Citations\n[1] [BigQuery Public Dataset: Stack Overflow Posts Privilege Wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_privilege_wiki)", "tables/posts_questions": "This table, part of the public [Stack Overflow dataset](/datasets/stackoverflow.md), contains comprehensive data for every question asked on the Stack Overflow platform. Each row represents a unique question post, including its content, metadata such as creation and last activity dates, view counts, and score. It serves as a primary resource for analyzing user activity, popular topics, and the overall dynamics of the Q&A community. The table can be joined with other tables like [posts_answers](/tables/posts_answers.md) to link questions to their corresponding answers, or with [users](/tables/users.md) to retrieve information about the question\\'s author.\n\n# Schema\nThe schema contains fields related to the question\\'s content, status, and associated user data. Key fields include `id` for unique identification, `title` and `body` for the question\\'s content, `creation_date` for when the question was posted, and `tags` for categorizing the question.\n\n- `id` (INTEGER) - Unique identifier for the post.\n- `title` (STRING) - The title of the question (for `PostTypeId = 1`).\n- `body` (STRING) - The body of the post, as rendered HTML, not Markdown.\n- `accepted_answer_id` (INTEGER) - The ID of the accepted answer; present only if `PostTypeId = 1`.\n- `answer_count` (INTEGER) - The number of undeleted answers; present only if `PostTypeId = 1`.\n- `closed_date` (TIMESTAMP) - The date and time the post was closed; present only if the post is closed.\n- `comment_count` (INTEGER) - The number of comments.\n- `community_owned_date` (TIMESTAMP) - The date and time the post became community wiki\\'d.\n- `creation_date` (TIMESTAMP) - The date and time the post was created.\n- `favorite_count` (INTEGER) - The number of times the post has been favorited. Nullable.\n- `last_activity_date` (TIMESTAMP) - The datetime of the post\\'s most recent activity.\n- `last_edit_date` (TIMESTAMP) - The date and time of the most recent edit to the post.\n- `last_editor_display_name` (STRING) - The display name of the last editor. Nullable.\n- `last_editor_user_id` (INTEGER) - The ID of the last editor user. Nullable.\n- `owner_display_name` (STRING) - The display name of the post owner. Nullable.\n- `owner_user_id` (INTEGER) - The ID of the post owner. Present only if user has not been deleted; always -1 for tag wiki entries.\n- `parent_id` (STRING) - Parent ID. For questions (`PostTypeId = 1`), this field is typically NULL. It is mainly used for answers (`PostTypeId = 2`) to link to their parent question.\n- `post_type_id` (INTEGER) - The type of the post. For questions, this is `1`. See the [Post Type IDs reference](/references/post_type_ids.md) for possible values.\n- `score` (INTEGER) - The score of the post; generally non-zero for Questions, Answers, and Moderator Nominations.\n- `tags` (STRING) - The tags associated with the question (for `PostTypeId = 1`).\n- `view_count` (INTEGER) - The number of times the post has been viewed. Nullable.\n- `content_license` (STRING) - Indicates the Creative Commons license under which the content is licensed.\n\n# Common query patterns\n1. **Find the most viewed questions in a specific tag:**\n   ```sql\n   SELECT\n     title,\n     view_count\n   FROM\n     `bigquery-public-data.stackoverflow.posts_questions`\n   WHERE\n     tags LIKE \\'%<your-tag>%'\n   ORDER BY\n     view_count DESC\n   LIMIT 10\n   ```\n2. **Count questions posted per year:**\n   ```sql\n   SELECT\n     EXTRACT(YEAR FROM creation_date) AS year,\n     COUNT(id) AS question_count\n   FROM\n     `bigquery-public-data.stackoverflow.posts_questions`\n   GROUP BY\n     year\n   ORDER BY\n     year DESC\n   ```\n3. **Get questions with accepted answers and their answer count:**\n   ```sql\n   SELECT\n     id,\n     title,\n     accepted_answer_id,\n     answer_count\n   FROM\n     `bigquery-public-data.stackoverflow.posts_questions`\n   WHERE\n     accepted_answer_id IS NOT NULL\n   ORDER BY\n     answer_count DESC\n   LIMIT 5\n   ```\n\n# Citations\n[1] [posts_questions](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_questions)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/posts_tag_wiki": "The `posts_tag_wiki` table in the BigQuery Stack Overflow dataset stores detailed wiki descriptions for various [tags](/tables/tags.md) used on the platform. Each row represents a wiki entry associated with a specific tag, providing comprehensive information about its purpose, usage, and related concepts. This table is useful for understanding the context and definition of tags, which helps in categorizing questions and answers accurately. The `body` field contains the rich text content of the wiki. This table is a valuable resource for anyone analyzing the structure and content organization of [Stack Overflow](/datasets/stackoverflow.md). The grain of this table is one row per tag wiki entry.\n\n# Schema\n- `id`: INTEGER, Unique identifier for the tag wiki post.\n- `title`: STRING, Title of the tag wiki (often null for actual tag wiki posts).\n- `body`: STRING, The main content of the tag wiki, containing detailed description.\n- `accepted_answer_id`: STRING, ID of the accepted answer, if applicable.\n- `answer_count`: STRING, Number of answers.\n- `comment_count`: INTEGER, Number of comments on the post.\n- `community_owned_date`: TIMESTAMP, Date when the post became community-owned.\n- `creation_date`: TIMESTAMP, Date and time the post was created.\n- `favorite_count`: STRING, Number of times the post has been favorited.\n- `last_activity_date`: TIMESTAMP, Date and time of the last activity on the post.\n- `last_edit_date`: TIMESTAMP, Date and time of the last edit to the post.\n- `last_editor_display_name`: STRING, Display name of the last editor.\n- `last_editor_user_id`: INTEGER, User ID of the last editor.\n- `owner_display_name`: STRING, Display name of the post owner.\n- `owner_user_id`: INTEGER, User ID of the post owner.\n- `parent_id`: STRING, ID of the parent post, if applicable.\n- `post_type_id`: INTEGER, Identifier for the type of post (e.g., 5 for Tag Wiki).\n- `score`: INTEGER, The score of the post.\n- `tags`: STRING, Tags associated with the post (usually null for tag wiki posts, as the wiki *is* for a tag).\n- `view_count`: STRING, Number of views the post has received.\n\n# Common query patterns\n\nTo retrieve the body of a specific tag wiki:\n```sql\nSELECT\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki`\n  WHERE\n    id = 5046395\n```\n\nTo find tag wiki entries created after a certain date:\n```sql\nSELECT\n    id,\n    creation_date,\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki`\n  WHERE\n    creation_date > '2020-01-01'\n  LIMIT 100\n```\n\nTo count the number of tag wiki entries per creation year:\n```sql\nSELECT\n    EXTRACT(YEAR FROM creation_date) AS creation_year,\n    COUNT(id) AS number_of_wikis\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki`\n  GROUP BY\n    creation_year\n  ORDER BY\n    creation_year DESC\n```\n\n# Citations\n[1] [BigQuery Public Dataset: Stack Overflow Posts Tag Wiki](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki)", "tables/posts_tag_wiki_excerpt": "The `posts_tag_wiki_excerpt` table in the BigQuery Stack Overflow public dataset contains summary information, or excerpts, from the wiki pages associated with various tags. Each row represents a single tag wiki excerpt, providing details such as the excerpt's body, its associated tag, and metadata about its creation and last modification. This table is useful for understanding the brief descriptions provided for tags on the Stack Overflow platform, offering quick insights into what a particular tag represents.\n\n# Schema\n- `id`: Unique identifier for the tag wiki excerpt.\n- `title`: Title of the tag wiki excerpt.\n- `body`: The content of the tag wiki excerpt.\n- `accepted_answer_id`: (STRING)\n- `answer_count`: (STRING)\n- `comment_count`: Number of comments on the excerpt.\n- `community_owned_date`: When the excerpt became community-owned.\n- `creation_date`: When the excerpt was created.\n- `favorite_count`: (STRING)\n- `last_activity_date`: Last activity date on the excerpt.\n- `last_edit_date`: Last edit date of the excerpt.\n- `last_editor_display_name`: Display name of the last editor.\n- `last_editor_user_id`: User ID of the last editor.\n- `owner_display_name`: Display name of the owner.\n- `owner_user_id`: User ID of the owner.\n- `parent_id`: (STRING)\n- `post_type_id`: Type of post.\n- `score`: Score of the excerpt.\n- `tags`: The tag associated with this wiki excerpt (e.g., `<python>`).\n- `view_count`: (STRING)\n\n# Common query patterns\n\n```sql\nSELECT\n    title,\n    body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`\n  WHERE\n    id = 12345\n```\n\n```sql\nSELECT\n    title,\n    tags\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`\n  WHERE\n    body LIKE '%python%'\n```\n\n```sql\nSELECT\n    title,\n    creation_date,\n    last_editor_display_name\n  FROM\n    `bigquery-public-data.stackoverflow.posts_tag_wiki_excerpt`\n  ORDER BY\n    creation_date DESC\n  LIMIT 10\n```\n\n# Citations\n[1] [BigQuery Public Data: Stack Overflow posts_tag_wiki_excerpt](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_tag_wiki_excerpt)", "tables/posts_wiki_placeholder": "This table serves as a placeholder for various wiki posts within the Stack Overflow community, containing community-maintained help, guidelines, and other informational content. Each row in this table represents a single wiki post, identified by a unique `id`. These posts often include details about election processes, help center articles, and general community guidelines.\n\n# Schema\n\n- `id`: INTEGER, Unique identifier for the wiki post.\n- `title`: STRING, Title of the wiki post.\n- `body`: STRING, The main content of the wiki post.\n- `accepted_answer_id`: STRING\n- `answer_count`: STRING\n- `comment_count`: INTEGER, Number of comments on the post.\n- `community_owned_date`: STRING\n- `creation_date`: TIMESTAMP, Timestamp when the post was created.\n- `favorite_count`: STRING\n- `last_activity_date`: TIMESTAMP, Timestamp of the last activity on the post.\n- `last_edit_date`: TIMESTAMP, Timestamp when the post was last edited.\n- `last_editor_display_name`: STRING\n- `last_editor_user_id`: INTEGER, User ID of the last editor.\n- `owner_display_name`: STRING\n- `owner_user_id`: INTEGER, User ID of the post owner.\n- `parent_id`: STRING\n- `post_type_id`: INTEGER, Type of the post (e.g., 7 for Wiki).\n- `score`: INTEGER, Score of the post.\n- `tags`: STRING, Tags associated with the post.\n- `view_count`: STRING\n\n# Common query patterns\n\n```sql\nSELECT\n    id, title, body\n  FROM\n    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`\n  WHERE\n    id = 8041931\n```\n\n```sql\nSELECT\n    id, title\n  FROM\n    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`\n  WHERE\n    body LIKE '%election process%'\n  LIMIT 100\n```\n\n```sql\nSELECT\n    EXTRACT(YEAR FROM creation_date) AS creation_year,\n    COUNT(id) AS post_count\n  FROM\n    `bigquery-public-data.stackoverflow.posts_wiki_placeholder`\n  GROUP BY\n    creation_year\n  ORDER BY\n    creation_year DESC\n```\n\n# Citations\n\n[1] [BigQuery Public Data: Stack Overflow posts_wiki_placeholder](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/posts_wiki_placeholder)", "tables/stackoverflow_posts": "This table, `stackoverflow_posts`, is a legacy table from the [stackoverflow](/datasets/stackoverflow.md) dataset in Google Cloud's BigQuery public datasets. It contains all posts (questions and answers) from Stack Overflow. **This table is deprecated and should not be used for new queries.** Users are advised to use the more specific and updated tables like [posts_questions](/tables/posts_questions.md) for questions and [posts_answers](/tables/posts_answers.md) for answers. Each row in this table represents a single post, with fields such as `id`, `title`, `body`, `creation_date`, and user information. The table contains approximately 31 million rows and over 31 GB of data.\n\n# Schema\n\n- `id`: Unique identifier for the post.\n- `title`: The title of the post (for questions).\n- `body`: The main content of the post.\n- `accepted_answer_id`: The ID of the accepted answer (for questions).\n- `answer_count`: Number of answers for a question.\n- `comment_count`: Number of comments on the post.\n- `community_owned_date`: Timestamp when the post became community-owned.\n- `creation_date`: Timestamp when the post was created.\n- `favorite_count`: Number of times the post has been favorited.\n- `last_activity_date`: Timestamp of the last activity on the post.\n- `last_edit_date`: Timestamp of the last edit.\n- `last_editor_display_name`: Display name of the last editor.\n- `last_editor_user_id`: User ID of the last editor.\n- `owner_display_name`: Display name of the post owner.\n- `owner_user_id`: User ID of the post owner.\n- `parent_id`: For answers, the ID of the question it answers.\n- `post_type_id`: Type of the post (e.g., 1 for question, 2 for answer).\n- `score`: The score of the post (upvotes - downvotes).\n- `tags`: Comma-separated tags associated with the post (for questions).\n- `view_count`: Number of views for the post (for questions).\n\n# Common query patterns\n\n```sql\nSELECT\n  id,\n  title,\n  creation_date\nFROM\n  `bigquery-public-data.stackoverflow.stackoverflow_posts`\nWHERE\n  creation_date BETWEEN '2008-01-01' AND '2008-01-31'\n  AND post_type_id = 1 -- Questions only\nLIMIT 100;\n```\n\n# Citations\n\n[1] [BigQuery Console - stackoverflow_posts](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/stackoverflow_posts)\n[2] [BigQuery Public Datasets - Stack Overflow](https://cloud.google.com/bigquery/public-data/stackoverflow)", "tables/tags": "This table, `tags`, from the [stackoverflow](/datasets/stackoverflow.md) dataset, provides a comprehensive list of all tags used across the Stack Overflow platform. Each row represents a unique tag, detailing its name, the total count of times it has been used, and references to its corresponding excerpt and wiki posts. This table is essential for understanding the topics and technologies discussed on Stack Overflow, allowing for analysis of tag popularity, trends, and content organization.\n\n# Schema\nThe `tags` table contains the following fields:\n- `id` (INTEGER) - Unique identifier for the tag.\n- `tag_name` (STRING) - The name of the tag (e.g., \\'python\\', \\'java\\', \\'javascript\\').\n- `count` (INTEGER) - The total number of times this tag has been used.\n- `excerpt_post_id` (INTEGER) - The ID of the Post that holds the excerpt text of the tag. Nullable.\n- `wiki_post_id` (INTEGER) - The ID of the Post that holds the wiki text of the tag. Nullable.\n- `is_moderator_only` (BOOLEAN) - Indicates if the tag can only be used by moderators.\n- `is_required` (BOOLEAN) - Indicates if the tag is required for certain post types.\n\n# Common query patterns\n\n1. Find the most popular tags:\n```sql\nSELECT\n    tag_name,\n    count\n  FROM\n    `bigquery-public-data.stackoverflow.tags`\n  GROUP BY\n    tag_name\n  ORDER BY\n    count DESC\n  LIMIT 10\n```\n\n2. Get details for a specific tag:\n```sql\nSELECT\n    id,\n    tag_name,\n    count,\n    excerpt_post_id,\n    wiki_post_id\n  FROM\n    `bigquery-public-data.stackoverflow.tags`\n  WHERE\n    tag_name = \\'python\\'\n```\n\n3. Join with posts to find posts related to a popular tag (example uses `posts_questions` which is related to `stackoverflow_posts`):\n```sql\nSELECT\n    p.title,\n    t.tag_name\n  FROM\n    `bigquery-public-data.stackoverflow.posts_questions` AS p,\n    `bigquery-public-data.stackoverflow.tags` AS t\n  WHERE\n    p.tags LIKE CONCAT(\\'%<\\', t.tag_name, \\'>%\\' )\n    AND t.tag_name = \\'javascript\\'\n  LIMIT 5\n```\n\n# Citations\n[1] [BigQuery Public Dataset: Stack Overflow Tags](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/tags)\n[2] [Stack Overflow Dataset on BigQuery](https://console.cloud.google.com/marketplace/details/stackoverflow/stackoverflow)\n[3] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/users": "This table, `users`, from the [stackoverflow](/datasets/stackoverflow.md) dataset, stores profiles of registered users on the Stack Overflow platform. Each row represents a unique user, identified by their `id`. The table includes details such as display name, creation date, last access date, reputation, and vote counts. It provides insights into the activity and characteristics of the Stack Overflow community members.\n\n# Schema\n\n*   `id` (INTEGER) - Unique identifier for the user.\n*   `display_name` (STRING) - The publicly visible name of the user.\n*   `about_me` (STRING) - User-provided free-form text about themselves. Nullable.\n*   `age` (INTEGER) - User-provided age. Nullable.\n*   `account_id` (INTEGER) - User\\'s Stack Exchange Network profile ID; NULL if the user has hidden this community in their profile. Nullable.\n*   `creation_date` (TIMESTAMP) - Timestamp when the user account was created.\n*   `email_hash` (STRING) - Gravatar email hash, now always NULL and will not appear as an attribute in the data dump XML. Nullable.\n*   `last_access_date` (TIMESTAMP) - Datetime user last loaded a page; updated every 30 minutes at most.\n*   `location` (STRING) - User-provided geographical location. Nullable.\n*   `reputation` (INTEGER) - The user\\'s reputation score.\n*   `up_votes` (INTEGER) - How many upvotes the user has cast.\n*   `down_votes` (INTEGER) - Total number of downvotes received by the user.\n*   `views` (INTEGER) - Number of times the user\\'s profile has been viewed.\n*   `profile_image_url` (STRING) - URL of the user\\'s profile picture. Nullable.\n*   `website_url` (STRING) - URL of the user\\'s personal website. Nullable.\n\n# Common query patterns\n\n1.  Find the top 10 users by reputation:\n    ```sql\n    SELECT\n      id,\n      display_name,\n      reputation\n    FROM\n      `bigquery-public-data.stackoverflow.users`\n    ORDER BY\n      reputation DESC\n    LIMIT 10\n    ```\n2.  Count users created per year:\n    ```sql\n    SELECT\n      EXTRACT(YEAR FROM creation_date) AS creation_year,\n      COUNT(id) AS user_count\n    FROM\n      `bigquery-public-data.stackoverflow.users`\n    GROUP BY\n      creation_year\n    ORDER BY\n      creation_year\n    ```\n3.  Find users with a high number of upvotes given:\n    ```sql\n    SELECT\n      display_name,\n      up_votes\n    FROM\n      `bigquery-public-data.stackoverflow.users`\n    WHERE\n      up_votes > 1000\n    ORDER BY\n      up_votes DESC\n    LIMIT 5\n    ```\n\n# Citations\n\n[1] [Stack Overflow Users Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/users)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)", "tables/votes": "This table, `votes`, from the [stackoverflow](/datasets/stackoverflow.md) dataset, records individual voting events on various posts (questions, answers, etc.) within the Stack Overflow platform. Each row represents a single vote, providing details such as the unique vote identifier, the timestamp of the vote, the ID of the post that received the vote, and the type of vote (e.g., upvote, downvote, favorite). This table is crucial for analyzing user engagement, post popularity, and the overall quality assessment mechanisms within the Stack Overflow community. The grain of this table is one row per vote event, and it captures historical voting data from its creation date up to the last modification.\n\n# Schema\n\n- `id` (INTEGER) - Unique identifier for the vote.\n- `post_id` (INTEGER) - The ID of the post to which the vote was cast. Links to `posts_questions` or `posts_answers` tables.\n- `vote_type_id` (INTEGER) - The type of vote. See the [Vote Types reference](/references/vote_types.md) for a comprehensive list.\n- `user_id` (INTEGER) - The ID of the user who cast the vote. Present only if `VoteTypeId` is `5` (Favorite/Bookmark) or `8` (BountyStart); -1 if user is deleted. Nullable. Links to the `users` table.\n- `creation_date` (TIMESTAMP) - The date when the vote was cast. Time data is purposefully removed to protect user privacy.\n- `bounty_amount` (INTEGER) - The amount of bounty associated with the vote. Present only if `VoteTypeId` is `8` (BountyStart) or `9` (BountyClose). Nullable.\n\n# Common query patterns\n\n```sql\n-- Get the total number of votes for each vote type\nSELECT\n  vote_type_id,\n  COUNT(id) AS total_votes\nFROM\n  `bigquery-public-data.stackoverflow.votes`\nGROUP BY\n  vote_type_id\nORDER BY\n  total_votes DESC;\n```\n\n```sql\n-- Find the top 10 most voted posts by summing up all votes (assuming all vote_type_ids contribute positively)\nSELECT\n  post_id,\n  COUNT(id) AS total_votes\nFROM\n  `bigquery-public-data.stackoverflow.votes`\nGROUP BY\n  post_id\nORDER BY\n  total_votes DESC\nLIMIT 10;\n```\n\n```sql\n-- Count votes over time\nSELECT\n  DATE(creation_date) AS vote_date,\n  COUNT(id) AS daily_votes\nFROM\n  `bigquery-public-data.stackoverflow.votes`\nWHERE\n  creation_date >= \\'2020-01-01\\' AND creation_date < \\'2021-01-01\\'\nGROUP BY\n  vote_date\nORDER BY\n  vote_date;\n```\n\n# Citations\n[1] [BigQuery Public Dataset: Stack Overflow Votes Table](https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/stackoverflow/tables/votes)\n[2] [Database schema documentation for the public data dump and SEDE](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede)"}, "types": ["BigQuery Dataset", "BigQuery Table", "Reference"], "palette": {"BigQuery Dataset": "#8b5cf6", "BigQuery Table": "#3b82f6", "Reference": "#10b981"}};
</script>
<script>
(function () {
  const bundle = window.BUNDLE;
  const bundleName = window.BUNDLE_NAME;
  document.title = `${bundleName} — OKF Viewer`;
  document.getElementById("bundle-name").textContent = bundleName;

  // Populate type filter
  const typeSelect = document.getElementById("filter-type");
  for (const t of bundle.types) {
    const opt = document.createElement("option");
    opt.value = t;
    opt.textContent = t;
    typeSelect.appendChild(opt);
  }

  // Build reverse-link index for backlinks
  const backlinks = {};
  for (const edge of bundle.edges) {
    const { source, target } = edge.data;
    (backlinks[target] ||= []).push(source);
  }

  // Look up node label/type by id
  const nodeIndex = {};
  for (const n of bundle.nodes) nodeIndex[n.data.id] = n.data;

  const cy = cytoscape({
    container: document.getElementById("graph"),
    elements: [...bundle.nodes, ...bundle.edges],
    style: [
      {
        selector: "node",
        style: {
          "background-color": "data(color)",
          "label": "data(label)",
          "color": "#0f172a",
          "font-size": 11,
          "text-valign": "bottom",
          "text-margin-y": 4,
          "text-wrap": "wrap",
          "text-max-width": 120,
          "width": "data(size)",
          "height": "data(size)",
          "border-width": 1,
          "border-color": "#0f172a",
        },
      },
      {
        selector: "node:selected",
        style: {
          "border-width": 3,
          "border-color": "#f59e0b",
        },
      },
      {
        selector: "edge",
        style: {
          "width": 1.5,
          "line-color": "#cbd5e1",
          "target-arrow-color": "#cbd5e1",
          "target-arrow-shape": "triangle",
          "curve-style": "bezier",
          "arrow-scale": 0.9,
        },
      },
      {
        selector: "edge:selected",
        style: {
          "line-color": "#f59e0b",
          "target-arrow-color": "#f59e0b",
          "width": 2.5,
        },
      },
      {
        selector: ".dim",
        style: { "opacity": 0.15 },
      },
    ],
    layout: { name: "cose", animate: false, padding: 30 },
    wheelSensitivity: 0.2,
  });

  cy.on("tap", "node", (evt) => showDetail(evt.target.id()));
  cy.on("tap", (evt) => {
    if (evt.target === cy) clearSelection();
  });

  document.getElementById("layout").addEventListener("change", (e) => {
    cy.layout({ name: e.target.value, animate: false, padding: 30 }).run();
  });

  document.getElementById("reset").addEventListener("click", () => {
    cy.fit(null, 30);
    clearSelection();
  });

  document.getElementById("search").addEventListener("input", (e) => {
    const q = e.target.value.trim().toLowerCase();
    if (!q) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      const d = n.data();
      const hay =
        (d.label || "").toLowerCase() + " " +
        d.id.toLowerCase() + " " +
        (d.tags || []).join(" ").toLowerCase();
      n.toggleClass("dim", !hay.includes(q));
    });
    cy.edges().forEach((edge) => {
      const src = edge.source();
      const tgt = edge.target();
      edge.toggleClass("dim", src.hasClass("dim") || tgt.hasClass("dim"));
    });
  });

  document.getElementById("filter-type").addEventListener("change", (e) => {
    const t = e.target.value;
    if (!t) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      n.toggleClass("dim", n.data("type") !== t);
    });
    cy.edges().forEach((edge) => {
      edge.toggleClass("dim", edge.source().hasClass("dim") || edge.target().hasClass("dim"));
    });
  });

  function clearSelection() {
    cy.elements().unselect();
    document.getElementById("detail-empty").hidden = false;
    document.getElementById("detail-content").hidden = true;
  }

  function showDetail(conceptId) {
    const data = nodeIndex[conceptId];
    if (!data) return;
    cy.elements().unselect();
    const node = cy.getElementById(conceptId);
    if (node) node.select();

    document.getElementById("detail-empty").hidden = true;
    const content = document.getElementById("detail-content");
    content.hidden = false;

    const chip = document.getElementById("detail-type");
    chip.textContent = data.type;
    chip.style.background = data.color;

    document.getElementById("detail-title").textContent = data.label;
    document.getElementById("detail-id").textContent = conceptId;
    document.getElementById("detail-description").textContent = data.description || "—";

    const resourceEl = document.getElementById("detail-resource");
    resourceEl.innerHTML = "";
    if (data.resource) {
      const a = document.createElement("a");
      a.href = data.resource;
      a.textContent = data.resource;
      a.target = "_blank";
      a.rel = "noopener";
      a.className = "external";
      resourceEl.appendChild(a);
    } else {
      resourceEl.textContent = "—";
    }

    const tagsEl = document.getElementById("detail-tags");
    tagsEl.innerHTML = "";
    if (data.tags && data.tags.length) {
      for (const t of data.tags) {
        const span = document.createElement("span");
        span.className = "tag";
        span.textContent = t;
        tagsEl.appendChild(span);
      }
    } else {
      tagsEl.textContent = "—";
    }

    const body = bundle.bodies[conceptId] || "";
    const html = marked.parse(body, { breaks: false, gfm: true });
    const bodyEl = document.getElementById("detail-body");
    bodyEl.innerHTML = html;
    rewriteInternalLinks(bodyEl);

    const bl = backlinks[conceptId] || [];
    const blSection = document.getElementById("detail-backlinks");
    const blList = document.getElementById("backlinks-list");
    blList.innerHTML = "";
    if (bl.length) {
      blSection.hidden = false;
      for (const src of bl) {
        const li = document.createElement("li");
        const a = document.createElement("a");
        a.textContent = nodeIndex[src]?.label || src;
        a.dataset.target = src;
        a.addEventListener("click", () => showDetail(src));
        li.appendChild(a);
        const muted = document.createElement("span");
        muted.className = "muted";
        muted.textContent = ` (${src})`;
        li.appendChild(muted);
        blList.appendChild(li);
      }
    } else {
      blSection.hidden = true;
    }

    cy.animate({ center: { eles: node }, zoom: Math.max(cy.zoom(), 1.0) }, { duration: 200 });
  }

  function rewriteInternalLinks(root) {
    root.querySelectorAll("a[href]").forEach((a) => {
      const href = a.getAttribute("href");
      if (!href) return;
      if (href.startsWith("/") && href.endsWith(".md")) {
        const target = href.slice(1, -3);
        if (nodeIndex[target]) {
          a.className = "internal";
          a.setAttribute("href", "javascript:void(0)");
          a.addEventListener("click", (e) => {
            e.preventDefault();
            showDetail(target);
          });
          return;
        }
      }
      a.className = "external";
      a.setAttribute("target", "_blank");
      a.setAttribute("rel", "noopener");
    });
  }

  // Auto-show the first node (a dataset if available, else first concept)
  const initial =
    bundle.nodes.find((n) => n.data.type === "BigQuery Dataset") ||
    bundle.nodes[0];
  if (initial) showDetail(initial.data.id);
})();

</script>
</body>
</html>


================================================
FILE: okf\LICENSE.md
================================================

                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/

   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

   1. Definitions.

      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.

      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.

      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.

      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.

      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.

      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).

      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.

      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."

      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.

   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.

   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:

      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and

      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and

      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and

      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.

      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.

   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.

   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.

   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.

   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.

   END OF TERMS AND CONDITIONS

   APPENDIX: How to apply the Apache License to your work.

      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

   Copyright [yyyy] [name of copyright owner]

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.



================================================
FILE: okf\README.md
================================================
# Open Knowledge Format (OKF)

### 📖 [Read the Open Knowledge Format v0.1 specification → SPEC.md](SPEC.md)

> **This repository is primarily about the [Open Knowledge Format
> (OKF)](SPEC.md).**
>
> OKF is a **universal, vendor-neutral format** for representing knowledge
> as plain markdown files with YAML frontmatter. It is **not tied to any
> particular agent, framework, model provider, or serving system**. The
> goal is simple:
>
> - **Anyone can produce** OKF — humans authoring by hand, agents built on
>   any framework (Google ADK, LangChain, custom), export pipelines from
>   existing catalogs (Dataplex, Unity Catalog, Collibra, …), or scripts
>   walking a database.
> - **Anyone can serve and consume** OKF — a static file server, a
>   knowledge-management UI (Obsidian, Notion, MkDocs), an LLM loading
>   files into context, a search index, or a graph viewer like the one
>   bundled in this repo.
>
> The agent below is a **proof of concept** demonstrating *one* way to
> produce OKF bundles automatically. The format itself is the
> contribution; this agent and the visualizer exist to make the format
> tangible at both ends — production and consumption.
>
> **See OKF in practice** — three ready-to-browse bundles produced by this
> agent, checked into [`bundles/`](bundles/):
>
> - [`bundles/ga4/`](bundles/ga4/) — GA4 e-commerce dataset
>   ([viz.html](bundles/ga4/viz.html))
> - [`bundles/stackoverflow/`](bundles/stackoverflow/) — Stack Overflow
>   public dataset ([viz.html](bundles/stackoverflow/viz.html))
> - [`bundles/crypto_bitcoin/`](bundles/crypto_bitcoin/) — Bitcoin
>   blocks/transactions ([viz.html](bundles/crypto_bitcoin/viz.html))

## Why OKF?

OKF represents catalog knowledge as plain markdown files with YAML
frontmatter, organized in a directory hierarchy. That choice unlocks a few
properties that are hard to get from a service-owned metadata store:

- **Human- and agent-readable.** No SDK or query language stands between a
  reader and the content. An engineer can `cat` a concept; an LLM can ingest
  it verbatim into context.
- **Version-controllable out of the box.** Bundles live in git. Pull
  requests, line-by-line diffs, blame, and review workflows just work —
  knowledge curation becomes a normal software-engineering activity.
- **Portable and lock-in free.** A bundle is a directory. Ship it as a
  tarball, host it in any repo, mount it from any filesystem, or sync it to
  any system that speaks files. No proprietary API stands between you and
  your metadata.
- **Mixes structured and unstructured data deliberately.** Use frontmatter
  for the few fields you want to query, filter, or index on (`type`,
  `resource`, `tags`, `timestamp`); use the markdown body for the prose,
  schemas, and example queries that LLMs and humans actually read.
- **Minimally opinionated, freely extensible.** A small set of required
  keys ensures interoperability, but bundles can carry arbitrary extra
  frontmatter keys and arbitrary body sections without breaking
  consumers.
- **Composes with existing tooling.** Many knowledge tools — Notion,
  Obsidian, MkDocs, Hugo, Jekyll — already speak markdown plus YAML
  frontmatter, so bundles can be browsed, edited, or rendered without
  custom UI.
- **Progressive disclosure built in.** Auto-generated `index.md` files
  let an agent or human navigate the hierarchy one level at a time
  instead of loading the entire bundle into context.
- **Graph-shaped, not just tree-shaped.** Concepts link to each other via
  normal markdown links, expressing relationships richer than the
  parent/child implied by the directory layout.

The net effect is that reference agents, consumption agents, and humans
collaborate on the same artifacts in the same way they already collaborate
on source code.

## Install

```
python3.13 -m venv .venv
.venv/bin/pip install --index-url https://pypi.org/simple/ -e .[dev]
```

## Credentials

- BigQuery: `gcloud auth application-default login` plus a project for billing
  (`gcloud config set project <id>`). Public datasets are readable, but the
  caller's project is billed for query bytes.
- Gemini: set `GEMINI_API_KEY` (AI Studio) **or** use Vertex AI by setting
  `GOOGLE_GENAI_USE_VERTEXAI=true`, `GOOGLE_CLOUD_PROJECT=<id>`, and
  `GOOGLE_CLOUD_LOCATION=<region>`.

## How the reference agent works

The reference agent runs in two passes. The **BQ pass** writes one OKF
doc per concept the source advertises, using BigQuery metadata alone.
The **web pass** then runs the LLM as its own crawler: it receives a
list of seed URLs (provided via `--web-seed` or `--web-seed-file`),
fetches the seeds via the `fetch_url` tool, and decides which outbound
links are worth following based on whether they look like authoritative
documentation for the existing concepts. For each page it fetches, the
agent chooses to (a) enrich one or more existing concept docs, (b) mint
a standalone `references/<slug>` doc, or (c) skip. A hard
`--web-max-pages` cap and a same-domain allowed-hosts filter
(configurable via `--web-allowed-host`) are enforced inside the tool,
so the agent cannot overrun. Use `--no-web` to skip the web pass.

## Run

Minimum invocation — point at a BigQuery dataset and a bundle output
directory. Seeds for the web pass are explicit; omit them (or pass
`--no-web`) to run BQ-only:

```
.venv/bin/python -m reference_agent enrich \
    --source bq \
    --dataset <project>.<dataset> \
    --web-seed-file <path/to/seeds.txt> \
    --out ./bundles/<name>
```

Iterate on a single concept by adding `--concept <type>/<name>` (e.g.
`--concept tables/events_`); repeatable.

## Samples

Each sample pairs a **recipe** (`samples/<name>/`, with the seed URLs and
exact `enrich` command) with the **produced bundle** (`bundles/<name>/`)
that the recipe generated. Open the recipe to reproduce; open the bundle
to browse the result directly.

- **GA4 Google Merchandise Store** — public e-commerce dataset, seeded
  with canonical GA4 BigQuery Export documentation URLs.
  · [recipe](samples/ga4_merch_store/README.md)
  · [bundle](bundles/ga4/)
  · [viz.html](bundles/ga4/viz.html)
- **Stack Overflow** — public dataset (mirror of the Stack Exchange Data
  Dump), seeded with the community's canonical schema references.
  Exercises multi-concept enrichment from cross-cutting docs pages.
  · [recipe](samples/stackoverflow/README.md)
  · [bundle](bundles/stackoverflow/)
  · [viz.html](bundles/stackoverflow/viz.html)
- **Bitcoin (crypto)** — public dataset (blocks, transactions, inputs,
  outputs) from the `bitcoin-etl` pipeline. Exercises cross-table
  foreign-key relationships in prose.
  · [recipe](samples/crypto_bitcoin/README.md)
  · [bundle](bundles/crypto_bitcoin/)
  · [viz.html](bundles/crypto_bitcoin/viz.html)

## Visualize

The `visualize` subcommand renders any OKF bundle as a **self-contained
interactive HTML file** — one file, no backend, no install on the
viewing side. Open it in any modern browser, share it as an artifact,
host it on a static file server, or commit it next to the bundle (as
this repo does).

The viewer is itself a proof-of-concept *consumer* of OKF, mirroring
the way the reference agent is a proof-of-concept *producer*. OKF
bundles can be consumed by anything that reads markdown; this is just
one shape.

### What it shows

- A **force-directed graph** of every concept in the bundle, with
  colored nodes by type (datasets, tables, references, …) and directed
  edges drawn from each cross-link in the markdown bodies.
- A **detail panel** for the selected concept showing its frontmatter
  (description, resource link, tags) and its rendered markdown body —
  with internal `[…](/path/to/concept.md)` links rewired to navigate
  within the viewer instead of following the path.
- A **"Cited by" backlinks** list under each concept (computed from the
  reverse of the link graph).
- A **search box** (matches title, concept id, and tags), a **type
  filter**, and switchable graph layouts (cose / concentric /
  breadth-first / circle / grid).

### Generate

```
.venv/bin/python -m reference_agent visualize --bundle ./bundles/<name>
```

That writes `bundles/<name>/viz.html`. Flags:

| Flag           | Default                | Description                                 |
|----------------|------------------------|---------------------------------------------|
| `--bundle`     | *(required)*           | Bundle root directory.                      |
| `--out`        | `<bundle>/viz.html`    | Output HTML path.                           |
| `--name`       | bundle directory name  | Display name shown in the viewer header.    |

Example, writing the output somewhere else and overriding the header:

```
.venv/bin/python -m reference_agent visualize \
    --bundle ./bundles/crypto_bitcoin \
    --out /tmp/btc.html \
    --name "Bitcoin OKF"
```

### How it's built

The HTML embeds the bundle as a JSON blob and uses
[Cytoscape.js](https://js.cytoscape.org/) for the graph and
[marked](https://marked.js.org/) for in-browser markdown rendering,
both loaded from a CDN. No data leaves the page; the bundle is parsed
once at generation time and serialized into the file.

## Tests

```
.venv/bin/pytest
```


================================================
FILE: okf\samples\crypto_bitcoin\README.md
================================================
# Bitcoin public dataset sample

Runs the reference agent against the public
`bigquery-public-data.crypto_bitcoin` dataset (blocks, transactions,
inputs, outputs — produced by the open-source `bitcoin-etl` pipeline)
and seeds the web pass with the canonical schema source and the
foundational Google Cloud blockchain-on-BigQuery announcement.

This sample contrasts with GA4 (single denormalized events table) and
Stack Overflow (many independent entities) by exercising a **small set
of tightly related fact tables** where each row in `transactions`
references rows in `blocks`, `inputs`, and `outputs`. Good for seeing
how the agent surfaces cross-table foreign-key relationships in prose.

## Prerequisites

- Install the agent (from the repo root):
  ```
  python3.13 -m venv .venv
  .venv/bin/pip install --index-url https://pypi.org/simple/ -e .[dev]
  ```
- BigQuery access:
  ```
  gcloud auth application-default login
  gcloud config set project <your-billing-project>
  ```
  Public datasets are readable, but the caller's project is billed for
  query bytes. The `crypto_bitcoin` tables are very large
  (`transactions` is ~hundreds of GB) — keep `--web-max-pages` modest
  while iterating and prefer `--concept` for smoke runs.
- Gemini credentials — either `GEMINI_API_KEY` (AI Studio) **or** Vertex
  AI (`GOOGLE_GENAI_USE_VERTEXAI=true`, `GOOGLE_CLOUD_PROJECT=<id>`,
  `GOOGLE_CLOUD_LOCATION=<region>`).

## Run

```
.venv/bin/python -m reference_agent enrich \
    --source bq \
    --dataset bigquery-public-data.crypto_bitcoin \
    --web-seed-file samples/crypto_bitcoin/seeds.txt \
    --out ./bundles/crypto_bitcoin
```

To iterate on a single concept, add `--concept tables/transactions`.
To skip the web pass, add `--no-web`. To raise or lower the web budget,
use `--web-max-pages N` (default 100).

## What you get

A bundle under `./bundles/crypto_bitcoin/` with one OKF doc per BQ
concept (dataset + each table), augmented and cross-linked with
reference docs minted from the seeded blockchain-etl and Google Cloud
pages, plus an auto-generated `index.md` at each directory level.


================================================
FILE: okf\samples\crypto_bitcoin\seeds.txt
================================================
# Seed URLs for the Bitcoin public BigQuery dataset
# (bigquery-public-data.crypto_bitcoin).
#
# The BQ dataset is produced by the open-source bitcoin-etl pipeline and
# mirrors its schema (blocks, transactions, inputs, outputs).
#
# Each non-comment line is one URL. The web-ingestion agent crawls outward
# from these by following links it judges relevant; same-domain by default.

# Canonical schema source — the bitcoin-etl repo's README documents the
# blocks / transactions / transaction_input / transaction_output schemas
# that map directly onto the crypto_bitcoin tables.
https://github.com/blockchain-etl/bitcoin-etl

# Foundational Google Cloud blog post on blockchain analytics in BigQuery.
# Note: it predates the crypto_bitcoin dataset name and references the
# older bitcoin_blockchain dataset, but provides authoritative context
# and links to newer material.
https://cloud.google.com/blog/products/gcp/bitcoin-in-bigquery-blockchain-analytics-on-public-data


================================================
FILE: okf\samples\ga4_merch_store\README.md
================================================
# GA4 Google Merchandise Store sample

Runs the reference agent against the public
`bigquery-public-data.ga4_obfuscated_sample_ecommerce` dataset (a GA4 export
from the Google Merchandise Store) and seeds the web pass with canonical GA4
BigQuery Export documentation URLs.

## Prerequisites

- Install the agent (from the repo root):
  ```
  python3.13 -m venv .venv
  .venv/bin/pip install --index-url https://pypi.org/simple/ -e .[dev]
  ```
- BigQuery access:
  ```
  gcloud auth application-default login
  gcloud config set project <your-billing-project>
  ```
  Public datasets are readable, but the caller's project is billed for
  query bytes.
- Gemini credentials — either `GEMINI_API_KEY` (AI Studio) **or** Vertex
  AI (`GOOGLE_GENAI_USE_VERTEXAI=true`, `GOOGLE_CLOUD_PROJECT=<id>`,
  `GOOGLE_CLOUD_LOCATION=<region>`).

## Run

```
.venv/bin/python -m reference_agent enrich \
    --source bq \
    --dataset bigquery-public-data.ga4_obfuscated_sample_ecommerce \
    --web-seed-file samples/ga4_merch_store/seeds.txt \
    --out ./bundles/ga4_merch_store
```

To iterate on a single concept, add `--concept tables/events_`. To skip
the web pass, add `--no-web`. To raise or lower the web budget, use
`--web-max-pages N` (default 100).

## What you get

A bundle under `./bundles/ga4_merch_store/` with one OKF doc per BQ
concept (dataset + tables), optionally augmented and cross-linked with
reference docs minted from the seeded GA4 documentation pages, plus an
auto-generated `index.md` at each directory level.


================================================
FILE: okf\samples\ga4_merch_store\seeds.txt
================================================
# Seed URLs for the GA4 Google Merchandise Store anchor dataset
# (bigquery-public-data.ga4_obfuscated_sample_ecommerce).
#
# Each non-comment line is one URL. The web-ingestion agent crawls outward
# from these by following links it judges relevant; same-domain by default.

# GA4 BigQuery Export — top-level overview and index
https://support.google.com/analytics/answer/7029846

# GA4 BigQuery Export — schema reference (events, items, params)
https://support.google.com/analytics/answer/7029846?hl=en

# GA4 BigQuery cookbook (example queries by use case)


================================================
FILE: okf\samples\stackoverflow\README.md
================================================
# Stack Overflow public dataset sample

Runs the reference agent against the public
`bigquery-public-data.stackoverflow` dataset (a mirror of the Stack
Exchange Data Dump for Stack Overflow — `posts_questions`,
`posts_answers`, `users`, `votes`, `comments`, `badges`, `tags`,
`post_history`, `post_links`, ...) and seeds the web pass with the
canonical schema references maintained by the Stack Exchange community.

This sample contrasts with the GA4 sample by exercising **multi-concept
enrichment**: a single schema-docs page typically describes several tables
(`posts_questions` + `posts_answers` + `users`), so the web agent often
updates more than one concept per fetched page.

## Prerequisites

- Install the agent (from the repo root):
  ```
  python3.13 -m venv .venv
  .venv/bin/pip install --index-url https://pypi.org/simple/ -e .[dev]
  ```
- BigQuery access:
  ```
  gcloud auth application-default login
  gcloud config set project <your-billing-project>
  ```
  Public datasets are readable, but the caller's project is billed for
  query bytes. The `stackoverflow` tables are large — keep
  `--web-max-pages` modest while iterating.
- Gemini credentials — either `GEMINI_API_KEY` (AI Studio) **or** Vertex
  AI (`GOOGLE_GENAI_USE_VERTEXAI=true`, `GOOGLE_CLOUD_PROJECT=<id>`,
  `GOOGLE_CLOUD_LOCATION=<region>`).

## Run

```
.venv/bin/python -m reference_agent enrich \
    --source bq \
    --dataset bigquery-public-data.stackoverflow \
    --web-seed-file samples/stackoverflow/seeds.txt \
    --out ./bundles/stackoverflow
```

To iterate on a single concept, add `--concept tables/posts_questions`.
To skip the web pass, add `--no-web`. To raise or lower the web budget,
use `--web-max-pages N` (default 100).

## What you get

A bundle under `./bundles/stackoverflow/` with one OKF doc per BQ concept
(dataset + each table), augmented and cross-linked with reference docs
minted from the seeded Stack Exchange schema pages, plus an
auto-generated `index.md` at each directory level.


================================================
FILE: okf\samples\stackoverflow\seeds.txt
================================================
# Seed URLs for the Stack Overflow public BigQuery dataset
# (bigquery-public-data.stackoverflow).
#
# The BQ dataset mirrors the Stack Exchange Data Dump schema (Posts, Users,
# Votes, Comments, Badges, Tags, PostHistory, PostLinks, ...). These seeds
# point at the canonical schema references the community maintains.
#
# Each non-comment line is one URL. The web-ingestion agent crawls outward
# from these by following links it judges relevant; same-domain by default.

# Canonical schema documentation for the Stack Exchange Data Dump / SEDE
# (covers Posts, Users, Votes, Comments, Badges, Tags, etc.)
https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede

# Stack Exchange Data Explorer help — table descriptions and example queries
https://data.stackexchange.com/help

# Internet Archive page for the Stack Exchange data dump (upstream source
# of the BigQuery export; includes README with field semantics)
https://archive.org/details/stackexchange


================================================
FILE: okf\SPEC.md
================================================
# Open Knowledge Format (OKF)

**Version 0.1 — Draft**

OKF is an open, human- and agent-friendly format for representing
*knowledge* — the metadata, context, and curated insight that surrounds
data and systems. It is designed to be authored by people, generated by
agents, exchanged across organizations, and consumed by both.

The format is intentionally minimal: a directory of markdown files with
YAML frontmatter. There is no schema registry, no central authority, and
no required tooling. If you can `cat` a file, you can read OKF; if you
can `git clone` a repo, you can ship it.

---

## 1. Motivation

The space of knowledge representation for AI agents is evolving quickly,
and many incompatible conventions are emerging. OKF takes the position
that knowledge is best represented in commonly accessible, established
formats that are:

- **Readable** by humans without tooling.
- **Parseable** by agents without bespoke SDKs.
- **Diffable** in version control.
- **Portable** across tools, organizations, and time.

The format is minimally opinionated. It standardizes only the small set
of structural conventions needed to make a knowledge corpus
*self-describing* — anything beyond that is left to the producer.

### Goals

1. Define a universal format that **enrichment agents** can write into.
2. Inform how **consumption agents** should read and traverse it.
3. Facilitate **exchange** of knowledge across systems and organizations.
4. Standardize the small number of **required** fields that must be
   present for content to be meaningfully consumed.

### Non-goals

- Defining a fixed taxonomy of concept types.
- Prescribing storage, serving, or query infrastructure.
- Replacing domain-specific schemas (Avro, Protobuf, OpenAPI, etc.) —
  OKF *references* them; it does not subsume them.

---

## 2. Terminology

- **Knowledge Bundle** — A self-contained, hierarchical collection of
  knowledge documents. The unit of distribution.
- **Concept** — A single unit of knowledge within a bundle. Represented
  as one markdown document. May describe a tangible asset (a table, an
  API), an abstract idea (a metric, a business process), or anything in
  between.
- **Concept ID** — The path of the concept's file within the bundle,
  with the `.md` suffix removed. For example, `tables/users.md` has
  concept ID `tables/users`.
- **Frontmatter** — YAML metadata block delimited by `---` at the top of
  a markdown file.
- **Body** — Everything in the file after the frontmatter.
- **Link** — A standard markdown link from one concept to another, used
  to express relationships beyond the implicit parent/child hierarchy.
- **Citation** — A link from a concept to an external source that
  supports a claim in the body.

---

## 3. Bundle Structure

A bundle is a directory tree of markdown files. The directory structure
is independent of the domain — producers organize concepts however makes
sense for the knowledge being captured.

```
path/to/bundle/
├── index.md                      # Optional. Directory listing for progressive disclosure.
├── log.md                        # Optional. Chronological history of updates.
├── <concept>.md                  # A concept at the bundle root.
└── <subdirectory>/               # Subdirectories organize concepts into groups.
    ├── index.md
    ├── <concept>.md
    └── <subdirectory>/
        └── …
```

A bundle MAY be distributed as:

- A git repository (recommended — provides history, attribution, diffs).
- A tarball or zip archive of the directory.
- A subdirectory within a larger repository.

### 3.1 Reserved filenames

The following filenames have defined meaning at any level of the
hierarchy and MUST NOT be used for concept documents:

| Filename     | Purpose                                                |
|--------------|--------------------------------------------------------|
| `index.md`   | Directory listing. See §6.                             |
| `log.md`     | Update history. See §7.                                |

All other `.md` files are concept documents.

Tags themselves remain a first-class concept — see the `tags`
frontmatter field in §4.1. OKF does not specify a separate file format
for aggregating documents by tag; producers that want a tag-browsing
view can synthesize one at consumption time by scanning frontmatter.

---

## 4. Concept Documents

Every concept is a UTF-8 markdown file. It has two parts:

1. A **YAML frontmatter block**, delimited by `---` on its own line at
   the start of the file and a closing `---` on its own line.
2. A **markdown body**, containing free-form content.

### 4.1 Frontmatter

```yaml
---
type: <Type name>                  # REQUIRED
title: <Optional display name>
description: <Optional one-line summary>
resource: <Optional canonical URI for the underlying asset>
tags: [<tag>, <tag>, …]            # Optional
timestamp: <ISO 8601 datetime>     # Optional last-modified time
# … other producer-defined key/value pairs
---
```

**Required:**

- `type` — A short string identifying the kind of concept. Consumers
  use this for routing, filtering, and presentation. Example values:
  `BigQuery Table`, `BigQuery Dataset`, `API Endpoint`, `Metric`,
  `Playbook`, `Reference`.

  Type values are **not** registered centrally. Producers SHOULD pick
  values that are descriptive and self-explanatory; consumers MUST
  tolerate unknown types gracefully (typically by treating them as
  generic concepts).

**Recommended (in priority order):**

- `title` — Human-readable display name. If omitted, consumers MAY
  derive a title from the filename.
- `description` — A single sentence summarizing the concept. Used by
  `index.md` generators, search snippets, and previews.
- `resource` — A URI that uniquely identifies the underlying asset the
  concept describes. Absent for concepts that describe abstract ideas
  rather than physical resources.
- `tags` — A YAML list of short strings for cross-cutting categorization.
- `timestamp` — ISO 8601 datetime of last meaningful change.

**Extensions:** Producers MAY include any additional keys. Consumers
SHOULD preserve unknown keys when round-tripping and SHOULD NOT reject
documents with unrecognized fields.

### 4.2 Body

The body is standard markdown. Producers SHOULD favor structural
markdown — headings, lists, tables, fenced code blocks — over freeform
prose, since structure aids both human reading and agent retrieval.

There are no required body sections. The following section headings have
**conventional** meaning and SHOULD be used when applicable:

| Heading        | Purpose                                                |
|----------------|--------------------------------------------------------|
| `# Schema`     | Structured description of an asset's columns/fields.   |
| `# Examples`   | Concrete usage examples, often as fenced code blocks.  |
| `# Citations`  | External sources backing claims in the body. See §8.   |

### 4.3 Example: a concept bound to a resource

```markdown
---
type: BigQuery Table
title: Customer Orders
description: One row per completed customer order across all channels.
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders
tags: [sales, orders, revenue]
timestamp: 2026-05-28T14:30:00Z
---

# Schema

| Column        | Type      | Description                              |
|---------------|-----------|------------------------------------------|
| `order_id`    | STRING    | Globally unique order identifier.        |
| `customer_id` | STRING    | Foreign key into [customers](/tables/customers.md). |
| `total_usd`   | NUMERIC   | Order total in US dollars.               |
| `placed_at`   | TIMESTAMP | When the customer submitted the order.   |

# Joins

Joined with [customers](/tables/customers.md) on `customer_id`.

# Citations

[1] [BigQuery table schema](https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders)
```

### 4.4 Example: a concept not bound to a resource

```markdown
---
type: Playbook
title: Incident response — data freshness alert
description: Steps to triage a freshness alert on the orders pipeline.
tags: [oncall, incident]
timestamp: 2026-04-12T09:00:00Z
---

# Trigger

A freshness alert fires when `orders` lags more than 30 minutes behind
its expected SLA. See the [orders table](/tables/orders.md).

# Steps

1. Check the [ingestion job dashboard](https://example.com/dash).
2. …
```

---

## 5. Cross-linking

Concepts MAY link to other concepts using standard markdown links. Two
forms are supported:

### 5.1 Absolute (bundle-relative) links

Begin with `/`, interpreted relative to the bundle root.

```markdown
See the [customers table](/tables/customers.md) for the join key.
```

This is the **recommended** form because it is stable when documents are
moved within their subdirectory.

### 5.2 Relative links

Standard markdown relative paths.

```markdown
See the [neighboring concept](./other.md).
```

### 5.3 Link semantics

A link from concept A to concept B asserts a *relationship*. The
specific kind of relationship (parent/child, references, joins-with,
depends-on, etc.) is conveyed by the surrounding prose, not by the link
itself. Consumers that build a graph view typically treat all links as
directed edges of an untyped relationship.

Consumers MUST tolerate broken links — a link whose target does not
exist in the bundle is not malformed; it may simply represent
not-yet-written knowledge.

---

## 6. Index Files

An `index.md` file MAY appear in any directory, including the bundle
root. It enumerates the directory's contents to support **progressive
disclosure** — letting a human or agent see what is available before
opening individual documents.

Index files contain no frontmatter. The body uses one or more sections,
each grouping concepts under a heading:

```markdown
# Section / Group Heading

* [Title 1](relative-url-1) - short description of item 1
* [Title 2](relative-url-2) - short description of item 2

# Another Section

* [Subdirectory](subdir/) - short description of the subdirectory
```

Entries SHOULD include the description from the linked concept's
frontmatter. Producers MAY generate `index.md` automatically; consumers
MAY synthesize one on the fly when none is present.

---

## 7. Log Files (optional)

A `log.md` file MAY appear at any level of the hierarchy to record the
history of changes to that scope. The format is a flat list of
date-grouped entries, newest first:

```markdown
# Directory Update Log

## 2026-05-22
* **Update**: Added new BigQuery table reference for [Customer Metrics](/tables/customer-metrics.md).
* **Creation**: Established the [Dataplex Playbook](/playbooks/dataplex.md).

## 2026-05-15
* **Initialization**: Created foundational directory structure.
* **Update**: Added progressive-disclosure guidelines to the root [index](/index.md).
```

Date headings MUST use ISO 8601 `YYYY-MM-DD` form. Log entries are
prose; the leading bold word (`**Update**`, `**Creation**`,
`**Deprecation**`, etc.) is a convention, not a requirement.

---

## 8. Citations

When a concept's body makes claims sourced from external material,
those sources SHOULD be listed under a `# Citations` heading at the
bottom of the document, numbered:

```markdown
# Citations

[1] [BigQuery public dataset announcement](https://cloud.google.com/blog/products/data-analytics/...)
[2] [Internal data quality runbook](https://wiki.acme.internal/data/quality)
```

Citation links MAY be absolute URLs, bundle-relative paths, or paths
into a `references/` subdirectory that mirrors external material as
first-class OKF concepts.

---

## 9. Conformance

A bundle is **conformant** with OKF v0.1 if:

1. Every non-reserved `.md` file in the tree contains a parseable YAML
   frontmatter block.
2. Every frontmatter block contains a non-empty `type` field.
3. Every reserved filename (`index.md`, `log.md`) follows the structure
   described in §6 and §7 respectively when present.

Consumers SHOULD treat all other constraints as soft guidance. In
particular, consumers MUST NOT reject a bundle because of:

- Missing optional frontmatter fields.
- Unknown `type` values.
- Unknown additional frontmatter keys.
- Broken cross-links.
- Missing `index.md` files.

This permissive consumption model is intentional: OKF is meant to
remain useful as bundles grow, get refactored, and are partially
generated by agents.

---

## 10. Relationship to other formats

OKF is intentionally close to several established patterns:

- **LLM "wiki" repositories** that use markdown + frontmatter as
  agent-readable knowledge bases.
- **Personal knowledge tools** like Obsidian and Notion, which use
  hierarchical markdown with cross-links.
- **"Metadata as code"** approaches that store catalog metadata
  alongside source code rather than in a separate registry.

OKF differs primarily in being **specified** — pinning down the small
set of rules needed for interoperability without dictating tooling.

---

## 11. Versioning

This document specifies OKF version **0.1**. Future revisions will be
versioned in the form `<major>.<minor>`:

- A **minor** version bump introduces backward-compatible additions
  (new optional fields, new conventional section headings).
- A **major** version bump may make breaking changes (renaming required
  fields, changing reserved filenames).

Bundles MAY declare the OKF version they target by including
`okf_version: "0.1"` in a bundle-root `index.md` frontmatter block (the
only place frontmatter is permitted in an `index.md`). Consumers that
do not understand the declared version SHOULD attempt best-effort
consumption rather than refusing the bundle.

---

## Appendix A — Minimal example bundle

```
my_bundle/
├── index.md
├── datasets/
│   ├── index.md
│   └── sales.md
└── tables/
    ├── index.md
    ├── orders.md
    └── customers.md
```

`datasets/sales.md`:

```markdown
---
type: BigQuery Dataset
title: Sales
description: All sales-related tables for the retail business.
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales
tags: [sales]
timestamp: 2026-05-28T00:00:00Z
---

The sales dataset contains transactional tables, including
[orders](/tables/orders.md) and [customers](/tables/customers.md).
```

`tables/orders.md`:

```markdown
---
type: BigQuery Table
title: Orders
description: One row per completed customer order.
resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders
tags: [sales, orders]
timestamp: 2026-05-28T00:00:00Z
---

# Schema

| Column        | Type      | Description                  |
|---------------|-----------|------------------------------|
| `order_id`    | STRING    | Unique order identifier.     |
| `customer_id` | STRING    | FK to [customers](/tables/customers.md). |
| `total_usd`   | NUMERIC   | Order total in USD.          |

Part of the [sales dataset](/datasets/sales.md).
```


================================================
FILE: okf\src\reference_agent\agent.py
================================================
from __future__ import annotations

from importlib import resources

from google.adk import Agent
from google.adk.tools import FunctionTool

from reference_agent.tools.bundle_tools import read_existing_doc, write_concept_doc
from reference_agent.tools.source_tools import (
    list_concepts,
    read_concept_raw,
    sample_rows,
)
from reference_agent.tools.web_tools import fetch_url

DEFAULT_MODEL = "gemini-flash-latest"


def _load_prompt(filename: str) -> str:
    return (
        resources.files("reference_agent.prompts")
        .joinpath(filename)
        .read_text(encoding="utf-8")
    )


def build_bq_agent(model: str = DEFAULT_MODEL) -> Agent:
    return Agent(
        name="okf_bq_reference_agent",
        model=model,
        instruction=_load_prompt("reference_instruction.md"),
        tools=[
            FunctionTool(list_concepts),
            FunctionTool(read_concept_raw),
            FunctionTool(sample_rows),
            FunctionTool(read_existing_doc),
            FunctionTool(write_concept_doc),
        ],
    )


def build_web_agent(model: str = DEFAULT_MODEL) -> Agent:
    return Agent(
        name="okf_web_ingestion_agent",
        model=model,
        instruction=_load_prompt("web_ingestion_instruction.md"),
        tools=[
            FunctionTool(list_concepts),
            FunctionTool(read_concept_raw),
            FunctionTool(read_existing_doc),
            FunctionTool(write_concept_doc),
            FunctionTool(fetch_url),
        ],
    )


================================================
FILE: okf\src\reference_agent\bundle\document.py
================================================
from __future__ import annotations

from dataclasses import dataclass, field
from typing import Any

import yaml

REQUIRED_FRONTMATTER_KEYS = ("type", "title", "description", "timestamp")

_FRONTMATTER_DELIM = "---"


class OKFDocumentError(ValueError):
    pass


@dataclass
class OKFDocument:
    frontmatter: dict[str, Any] = field(default_factory=dict)
    body: str = ""

    @classmethod
    def parse(cls, text: str) -> "OKFDocument":
        lines = text.splitlines()
        if not lines or lines[0].strip() != _FRONTMATTER_DELIM:
            return cls(frontmatter={}, body=text)

        end_idx = None
        for i in range(1, len(lines)):
            if lines[i].strip() == _FRONTMATTER_DELIM:
                end_idx = i
                break
        if end_idx is None:
            raise OKFDocumentError("Unterminated YAML frontmatter block")

        fm_text = "\n".join(lines[1:end_idx])
        try:
            fm = yaml.safe_load(fm_text) or {}
        except yaml.YAMLError as e:
            raise OKFDocumentError(f"Invalid YAML in frontmatter: {e}") from e
        if not isinstance(fm, dict):
            raise OKFDocumentError("Frontmatter must be a YAML mapping")

        body = "\n".join(lines[end_idx + 1:])
        if body.startswith("\n"):
            body = body[1:]
        return cls(frontmatter=fm, body=body)

    def serialize(self) -> str:
        fm_text = yaml.safe_dump(
            self.frontmatter, sort_keys=False, allow_unicode=True
        ).rstrip()
        body = self.body if self.body.endswith("\n") else self.body + "\n"
        return f"{_FRONTMATTER_DELIM}\n{fm_text}\n{_FRONTMATTER_DELIM}\n\n{body}"

    def validate(self) -> None:
        missing = [k for k in REQUIRED_FRONTMATTER_KEYS if not self.frontmatter.get(k)]
        if missing:
            raise OKFDocumentError(
                f"Missing required frontmatter keys: {', '.join(missing)}"
            )


================================================
FILE: okf\src\reference_agent\bundle\index.py
================================================
from __future__ import annotations

from collections import defaultdict
from pathlib import Path
from typing import Callable

from reference_agent.bundle.document import OKFDocument
from reference_agent.bundle.synthesizer import synthesize_description

_INDEX_FILE = "index.md"
_FALLBACK_MODEL = "gemini-flash-latest"


def _load_doc(path: Path) -> OKFDocument | None:
    try:
        return OKFDocument.parse(path.read_text(encoding="utf-8"))
    except Exception:
        return None


def _build_index_text(entries: list[tuple[str, str, str, str]]) -> str:
    # entries: (type, title, relative_link, description)
    grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
    for typ, title, link, desc in entries:
        grouped[typ or "Other"].append((title, link, desc))

    sections: list[str] = []
    for typ in sorted(grouped):
        lines = [f"# {typ}", ""]
        for title, link, desc in sorted(grouped[typ], key=lambda e: e[0].lower()):
            suffix = f" - {desc}" if desc else ""
            lines.append(f"* [{title}]({link}){suffix}")
        sections.append("\n".join(lines))
    return "\n\n".join(sections) + "\n"


def _directories_to_index(bundle_root: Path) -> list[Path]:
    dirs: set[Path] = set()
    for md in bundle_root.rglob("*.md"):
        cur = md.parent
        while cur != bundle_root.parent:
            dirs.add(cur)
            if cur == bundle_root:
                break
            cur = cur.parent
    return sorted(dirs)


def regenerate_indexes(
    bundle_root: Path,
    *,
    model: str = _FALLBACK_MODEL,
    synthesize: Callable[..., str] = synthesize_description,
) -> list[Path]:
    bundle_root = Path(bundle_root)
    written: list[Path] = []
    if not bundle_root.exists():
        return written

    directories = sorted(
        _directories_to_index(bundle_root),
        key=lambda p: (-len(p.relative_to(bundle_root).parts), str(p)),
    )

    dir_descriptions: dict[Path, str] = {}

    for directory in directories:
        entries: list[tuple[str, str, str, str]] = []

        for child in sorted(directory.iterdir()):
            if child.name == _INDEX_FILE:
                continue
            if child.is_file() and child.suffix == ".md":
                doc = _load_doc(child)
                if doc is None:
                    continue
                fm = doc.frontmatter
                title = str(fm.get("title") or child.stem)
                desc = str(fm.get("description") or "")
                typ = str(fm.get("type") or "")
                entries.append((typ, title, child.name, desc))
            elif child.is_dir():
                desc = dir_descriptions.get(child, "")
                entries.append(("Subdirectories", child.name, f"{child.name}/{_INDEX_FILE}", desc))

        if not entries:
            continue

        index_path = directory / _INDEX_FILE
        index_path.write_text(_build_index_text(entries), encoding="utf-8")
        written.append(index_path)

        if directory == bundle_root:
            continue

        pairs = [(title, desc) for _, title, _, desc in entries]
        if len(pairs) == 1 and pairs[0][1]:
            dir_descriptions[directory] = pairs[0][1]
        else:
            rel = str(directory.relative_to(bundle_root))
            dir_descriptions[directory] = synthesize(rel, pairs, model=model)

    return written


================================================
FILE: okf\src\reference_agent\bundle\paths.py
================================================
from __future__ import annotations

import re
from pathlib import Path

_SEGMENT_RE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.\-]*")


def _validate_segment(seg: str) -> None:
    if not _SEGMENT_RE.fullmatch(seg):
        raise ValueError(f"Invalid concept id segment: {seg!r}")


def concept_id_to_path(bundle_root: Path, concept_id: tuple[str, ...]) -> Path:
    if not concept_id:
        raise ValueError("concept_id must have at least one segment")
    for seg in concept_id:
        _validate_segment(seg)
    *dirs, name = concept_id
    return bundle_root.joinpath(*dirs, f"{name}.md")


def path_to_concept_id(bundle_root: Path, path: Path) -> tuple[str, ...]:
    rel = path.relative_to(bundle_root).with_suffix("")
    return tuple(rel.parts)


def parse_concept_id(s: str) -> tuple[str, ...]:
    parts = tuple(p for p in s.split("/") if p)
    if not parts:
        raise ValueError(f"Empty concept id: {s!r}")
    for p in parts:
        _validate_segment(p)
    return parts


================================================
FILE: okf\src\reference_agent\bundle\synthesizer.py
================================================
from __future__ import annotations

import logging

log = logging.getLogger(__name__)

_PROMPT_TEMPLATE = """\
You are summarizing a directory in an Open Knowledge Format bundle in ONE \
sentence (max ~25 words).

Directory: {rel_path}
Contents:
{contents}

Write one sentence that names what this directory collectively contains. Be \
concrete and factual; do not editorialize. Output the sentence only — no \
preamble, no quotes, no trailing punctuation beyond a single period.
"""


def _fallback(children: list[tuple[str, str]]) -> str:
    titles = ", ".join(t for t, _ in children if t) or "no titled entries"
    return f"Contains {len(children)} entries: {titles}."


def synthesize_description(
    rel_path: str,
    children: list[tuple[str, str]],
    *,
    model: str,
) -> str:
    if not children:
        return ""
    contents = "\n".join(
        f"- {title}: {desc}" if desc else f"- {title}"
        for title, desc in children
    )
    prompt = _PROMPT_TEMPLATE.format(rel_path=rel_path, contents=contents)
    try:
        from google import genai

        client = genai.Client()
        response = client.models.generate_content(model=model, contents=prompt)
        text = (getattr(response, "text", None) or "").strip()
        if not text:
            return _fallback(children)
        return text.splitlines()[0].strip()
    except Exception as exc:
        log.warning("synthesize_description failed for %s: %s", rel_path, exc)
        return _fallback(children)


================================================
FILE: okf\src\reference_agent\bundle\__init__.py
================================================
from reference_agent.bundle.document import OKFDocument, REQUIRED_FRONTMATTER_KEYS
from reference_agent.bundle.index import regenerate_indexes
from reference_agent.bundle.paths import concept_id_to_path, path_to_concept_id

__all__ = [
    "OKFDocument",
    "REQUIRED_FRONTMATTER_KEYS",
    "concept_id_to_path",
    "path_to_concept_id",
    "regenerate_indexes",
]


================================================
FILE: okf\src\reference_agent\cli.py
================================================
from __future__ import annotations

import argparse
import logging
import sys
from pathlib import Path
from urllib.parse import urlparse

from reference_agent.agent import DEFAULT_MODEL
from reference_agent.bundle.paths import parse_concept_id
from reference_agent.runner import ReferenceRunner
from reference_agent.sources.bigquery import BigQuerySource

_SOURCES = ("bq",)


def _build_source(name: str, args: argparse.Namespace):
    if name == "bq":
        if not args.dataset:
            raise SystemExit("--dataset is required for --source bq")
        return BigQuerySource(
            dataset=args.dataset, billing_project=args.billing_project
        )
    raise SystemExit(f"Unknown source: {name}")


def _parse_seed_file(path: Path) -> list[str]:
    urls: list[str] = []
    text = path.read_text(encoding="utf-8")
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if line:
            urls.append(line)
    return urls


def _collect_seeds(args: argparse.Namespace) -> list[str]:
    if args.no_web:
        return []
    seeds: list[str] = []
    if args.web_seed:
        seeds.extend(args.web_seed)
    if args.web_seed_file:
        for p in args.web_seed_file:
            seeds.extend(_parse_seed_file(Path(p)))
    return _dedup_preserve_order(seeds)


def _dedup_preserve_order(items: list[str]) -> list[str]:
    seen: set[str] = set()
    out: list[str] = []
    for x in items:
        if x not in seen:
            seen.add(x)
            out.append(x)
    return out


def _parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(prog="reference-agent")
    sub = p.add_subparsers(dest="command", required=True)

    enrich = sub.add_parser(
        "enrich", help="Enrich concepts from a source into an OKF bundle."
    )
    enrich.add_argument("--source", choices=_SOURCES, required=True)
    enrich.add_argument(
        "--dataset",
        help="Source-specific identifier (for --source bq: 'project.dataset').",
    )
    enrich.add_argument(
        "--billing-project",
        help="Google Cloud project to bill for queries; "
        "defaults to ADC default.",
    )
    enrich.add_argument(
        "--out", required=True, type=Path, help="Bundle root directory."
    )
    enrich.add_argument(
        "--concept",
        action="append",
        default=None,
        help="Enrich only this concept id (e.g. 'tables/events_'). "
        "Repeatable.",
    )
    enrich.add_argument(
        "--model",
        default=DEFAULT_MODEL,
        help=f"Gemini model id (default: {DEFAULT_MODEL}).",
    )
    enrich.add_argument(
        "--web-seed",
        action="append",
        default=None,
        help="Seed URL for the web pass. Repeatable.",
    )
    enrich.add_argument(
        "--web-seed-file",
        action="append",
        default=None,
        help="Path to a file with one seed URL per line (# comments allowed). "
        "Repeatable.",
    )
    enrich.add_argument(
        "--web-max-pages",
        type=int,
        default=100,
        help="Hard cap on pages the web agent may fetch in one run (default 100).",
    )
    enrich.add_argument(
        "--web-allowed-host",
        action="append",
        default=None,
        help="Extra hostname the web agent may fetch beyond seed hostnames. "
        "Repeatable. Default: only seed hosts.",
    )
    enrich.add_argument(
        "--web-allowed-path-prefix",
        action="append",
        default=None,
        help="Only fetch URLs whose path starts with one of these prefixes "
        "(e.g. '/docs/'). Repeatable. Default: no path restriction.",
    )
    enrich.add_argument(
        "--web-denied-path-substring",
        action="append",
        default=None,
        help="Reject URLs whose path contains any of these substrings "
        "(e.g. '/login', '/pricing'). Repeatable.",
    )
    enrich.add_argument(
        "--web-max-depth",
        type=int,
        default=2,
        help="Hard cap on hop distance from any seed URL (default 2). "
        "Seeds are depth 0; their outbound links are depth 1; etc.",
    )
    enrich.add_argument(
        "--no-web",
        action="store_true",
        help="Skip the web pass entirely.",
    )
    enrich.add_argument("-v", "--verbose", action="store_true")

    viz = sub.add_parser(
        "visualize",
        help="Generate a self-contained HTML graph view of an OKF bundle.",
    )
    viz.add_argument(
        "--bundle", required=True, type=Path,
        help="Path to the bundle root directory.",
    )
    viz.add_argument(
        "--out", type=Path, default=None,
        help="Output HTML path (default: <bundle>/viz.html).",
    )
    viz.add_argument(
        "--name", default=None,
        help="Display name for the bundle (default: bundle directory name).",
    )
    return p


def main(argv: list[str] | None = None) -> int:
    args = _parser().parse_args(argv)
    logging.basicConfig(
        level=logging.INFO,
        format="%(message)s",
    )
    if getattr(args, "verbose", False):
        logging.getLogger("reference_agent").setLevel(logging.DEBUG)
    # Quiet chatty third-party loggers regardless of mode.
    for noisy in ("google", "google_genai", "google_adk", "urllib3", "httpx"):
        logging.getLogger(noisy).setLevel(logging.WARNING)

    if args.command == "visualize":
        from reference_agent.viewer import generate_visualization
        out = args.out or (args.bundle / "viz.html")
        stats = generate_visualization(args.bundle, out, bundle_name=args.name)
        print(
            f"Wrote {stats['concepts']} concept(s), "
            f"{stats['edges']} edge(s), "
            f"{stats['bytes']} bytes → {out}",
            file=sys.stderr,
        )
        return 0

    if args.command == "enrich":
        source = _build_source(args.source, args)
        seeds = _collect_seeds(args)
        allowed_hosts: set[str] | None = None
        if seeds:
            allowed_hosts = {urlparse(s).netloc for s in seeds if urlparse(s).netloc}
            if args.web_allowed_host:
                allowed_hosts |= set(args.web_allowed_host)
        runner = ReferenceRunner(
            source=source,
            bundle_root=args.out,
            model=args.model,
            web_seeds=seeds or None,
            web_max_pages=args.web_max_pages,
            web_allowed_hosts=allowed_hosts,
            web_allowed_path_prefixes=args.web_allowed_path_prefix,
            web_denied_path_substrings=args.web_denied_path_substring,
            web_max_depth=args.web_max_depth,
            verbose=args.verbose,
        )
        only = (
            [parse_concept_id(c) for c in args.concept] if args.concept else None
        )
        n = runner.enrich_all(only=only)
        web_note = f"; web pass used {len(seeds)} seed(s)" if seeds else "; web pass skipped"
        print(f"Enriched {n} concept(s) into {args.out}{web_note}", file=sys.stderr)
        return 0
    return 1


================================================
FILE: okf\src\reference_agent\prompts\reference_instruction.md
================================================
You are an enrichment agent that produces **Open Knowledge Format (OKF)**
documents from raw source metadata. Each invocation enriches exactly **one**
concept and finishes by calling `write_concept_doc` exactly once.

## Workflow

1. Call `read_existing_doc(concept_id)` to see whether a prior document exists.
   If it does, use it as a starting point and refine rather than rewrite.
2. Call `read_concept_raw(concept_id)` to get structured metadata (schema,
   partitioning, etc.).
3. Optionally call `sample_rows(concept_id, n=3)` if the metadata is sparse
   and a small data sample would help you describe the concept.
4. Call `list_concepts()` to learn what other concepts exist in the bundle.
   Use the result to weave cross-links into your prose (see "Cross-linking").
5. Compose an OKF document and call `write_concept_doc(concept_id, frontmatter,
   body)` exactly once. Do not call any tools after that.

## Frontmatter (YAML, required keys)

- `type`: the concept type, exactly as returned in the concept ref (e.g.
  `BigQuery Table`, `BigQuery Dataset`).
- `title`: a short human-readable display name.
- `description`: **one sentence** explaining what this concept is. This is
  used verbatim in auto-generated `index.md` files, so keep it tight and
  informative.
- `timestamp`: leave unset and the tool will fill in the current UTC time, or
  provide an ISO 8601 string yourself.
- `resource` (recommended when applicable): the URI of the underlying asset.
- `tags` (recommended): a comma-separated list or YAML list of useful search
  tags inferred from the metadata.

## Body sections

In this order:

1. A short prose description (1–3 paragraphs) of what this concept is, what it
   represents, and how it is typically used. For tables, describe the grain
   (one row per X), the time range, and any obfuscation or sampling caveats.
2. `# Schema` — a flattened, readable summary of fields. For nested RECORD
   fields, indent or table-format their sub-fields. Skip mode/type when they
   are obvious. Highlight repeated records explicitly.
3. `# Common query patterns` — 1 to 3 short SQL snippets, fenced as
   ```` ```sql ```` blocks, illustrating realistic usage of this asset.
4. `# Citations` — use the OKF format:

       [1] [Source Title](https://example.com/...)
       [2] [Another Source](https://example.com/...)

   Include this concept's `resource` value as the first entry (when present);
   follow it with any URLs that informed the description. Do not invent URLs;
   cite only sources you actually know.

## Cross-linking

When your prose naturally references another concept by name — a sibling
table, the parent dataset, a reference doc — link to it using a path
**relative to the current document's directory**, so the link resolves
correctly when the bundle is browsed as plain files (e.g. on GitHub).
The list of available targets comes from `list_concepts()` (workflow
step 4). Examples, written from a doc at `tables/<this_table>.md`:

- Sibling table: `[users](users.md)`
- Parent dataset from a table: `[dataset](../datasets/<slug>.md)`
- Reference doc: `[event parameters](../references/event_parameters.md)`

Rules:

- Use file-relative paths only. Never start a link with `/` (that breaks
  GitHub rendering), and don't use bare filenames that aren't actual
  siblings.
- Only link to ids returned by `list_concepts()`. Do not invent link targets.
- One link per concept mention per section is enough. Do not over-link.
- Do not link from headers, fenced code blocks, or schema field-name listings.
- Do not link the current doc to itself.

## Style

- Be concrete. Prefer concrete examples and concrete field names over generic
  hand-waving.
- Do not invent fields, partitions, or shard counts that are not in the raw
  metadata.
- Do not include preamble, apologies, or reasoning narration in the document
  body. The body must be valid markdown that a human or downstream agent can
  consume directly.


================================================
FILE: okf\src\reference_agent\prompts\web_ingestion_instruction.md
================================================
You are a web-ingestion agent that augments an existing **Open Knowledge
Format (OKF)** bundle with information from web pages. You drive your own
crawl: starting from a list of seed URLs, you decide which links are worth
following and what to do with each page you fetch.

## Inputs

The user message contains:
- A list of **seed URLs** to start from.
- A **max-pages budget** (a hard cap enforced by the `fetch_url` tool; you
  cannot exceed it).
- Optionally, a list of **allowed hosts**. By default only the hosts of the
  seed URLs are allowed.

## Workflow

1. Call `list_concepts()` once at the start to learn what concepts the
   bundle already has. You will route web findings against these.
2. For each seed URL, call `fetch_url(url)`. The result includes the page's
   markdown content and `links` — its outbound URLs.
3. From those links, pick a small handful that look like they lead to
   **authoritative documentation** on topics related to the existing
   concepts. Skip nav links, site footers, login pages, "About us",
   marketing pages, cookie/privacy notices, and anything obviously
   tangential. Call `fetch_url` on each selected link. Their results in
   turn contain more links, which you can also follow — recursively, with
   your judgment as the filter.
4. For **each page you fetch**, decide one of:
   - **Enrich existing concept(s)**. If the page describes a topic that an
     existing concept doc covers (e.g. a schema reference for a specific
     table), call `read_existing_doc(concept_id)` to read the current doc,
     then call `write_concept_doc(concept_id, frontmatter, body)` with the
     **augmented** doc. Augmentation is strict (see "Augmentation rules"
     below) — you must preserve the existing structure verbatim and add
     content within or alongside it. You may update multiple concepts from
     a single page.
   - **Mint a new reference concept** — only if the page meets all four
     of:
     1. **Topic shape**: it defines something *referenceable by name*
        from a primary concept doc. Allowed kinds: a business entity
        definition, a metric definition, an enum or status-code
        reference, a field/parameter glossary, a pricing/billing note,
        a units/timezone/identifier convention.
     2. **Not bundle-level meta**: it is NOT an overview, introduction,
        "getting started", quickstart, tutorial, walkthrough, release
        notes, changelog, roadmap, FAQ, or product landing page. If the
        page title or URL slug contains any of `overview`, `intro`,
        `getting-started`, `quickstart`, `tutorial`, `walkthrough`,
        `release-notes`, `changelog`, `roadmap`, `faq` — skip.
     3. **Citation test**: you can plausibly write a sentence in a
        primary concept doc of the form
        `See the [X reference](/references/x.md) for ...` where X is a
        concrete noun (an entity, a metric, an enum, a field set). If
        the best sentence you can write is "See the overview for
        context", it fails this test.
     4. **Reuse test**: at least two existing concepts would benefit
        from citing it, OR one existing concept needs it as
        load-bearing background that doesn't fit in its own doc.

     If all four hold: pick an id under `references/` (e.g.
     `references/event_parameters`), set `type: Reference`, set
     `resource` to this page's URL, call `write_concept_doc`, and
     cross-link from each related primary doc with a markdown link
     written **relative to the linking doc's directory**, e.g. from a
     `tables/<slug>.md` doc:
     `[Event parameters reference](../references/event_parameters.md)`.

     When in doubt, **skip**. A bundle with zero `references/` docs is
     fine; a bundle full of `references/overview` and
     `references/getting_started` is noise.
   - **Skip**. If the page is irrelevant, low-signal, or already covered,
     do nothing. Move on.
5. Stop when:
   - `fetch_url` returns `"max_pages reached"` — your budget is spent.
   - You have covered the relevant material on the seed sites and further
     fetches would have diminishing returns.

## Frontmatter conventions

When you write a doc — primary or reference — frontmatter must include at
minimum `type`, `title`, `description` (one sentence; used in `index.md`),
and `timestamp` (leave unset; the tool will fill it). For reference docs:

- `type`: `Reference`
- `resource`: the canonical source URL (the page you ingested)
- `tags`: a YAML list inferred from the page topic

## Augmentation rules

When you call `write_concept_doc` for a concept that **already has an
on-disk doc** (i.e. `read_existing_doc` returned non-null), the call is
an *augmentation*, not a rewrite. Treat the existing doc as the source of
truth and fold the web page into it. These rules are non-negotiable:

1. **Frontmatter — pass the complete dict, with existing values preserved:**
   `write_concept_doc` does a full replacement, not a patch — the
   `frontmatter` argument **must include every key** the existing doc had
   (`type`, `title`, `description`, `resource`, `tags`, etc.). Omitting a
   key drops it. The augmentation rule is about which *values* you keep,
   not which *keys* you send. Specifically:
   - Copy `type` verbatim from the existing frontmatter into your new dict.
   - Copy `title` verbatim. The web page's `<title>` is **not** the
     concept's title.
   - Copy `resource` verbatim. For a `BigQuery Table` doc the `resource`
     is the BigQuery REST URI; it must stay that. The web page URL goes
     in `# Citations`, never in `resource`.
   - For `tags`, pass the union of existing tags plus any new ones
     (merge, don't replace).
   - Leave `timestamp` unset (omit the key or set it to empty) so the
     tool refreshes it. This is the *only* key you may legitimately drop.
   - You may refine `description` if the web page surfaces a more
     accurate one-sentence summary; otherwise copy it verbatim.

2. **Body — every `#` heading in the existing body must appear in your
   new body**, in the same order, with the same wording. You may:
   - extend the prose under each heading,
   - add new bullets to existing lists (e.g. add fields to `# Schema`,
     not replace the list),
   - add new sub-sections (`##`) under existing top-level headings,
   - add brand-new top-level headings **after** the existing ones,
   - append the web page's URL to `# Citations`.
   You may not:
   - drop or rename any existing `#` heading,
   - replace the body wholesale with a topical rewrite of the web page,
   - shrink or rewrite the `# Schema` section for a `BigQuery Table` doc
     — the BQ pass populated it from real schema metadata; keep every
     field listing.

3. **If you cannot honor rule 2** because the web page is a fundamentally
   different topic (a query cookbook, a release notes page, a generic
   tutorial), do **not** call `write_concept_doc` for the existing
   concept. Either mint a `references/<slug>` doc and cross-link from the
   primary doc's prose, or skip the page.

## Required extractions: metrics, dimensions, join paths

When a fetched page contains any of the following content types, you
**must** capture them in the appropriate doc — these are the
highest-signal artifacts a web page can contribute and they are easy to
lose in a topical paraphrase. For each, the destination and required
shape are non-negotiable:

- **Aggregate metrics** (e.g. *daily active users*, *conversion rate*,
  *revenue per user*, *retention curve*). Capture the metric's name, a
  one-line definition, and the **concrete SQL expression** (e.g.
  `COUNT(DISTINCT user_pseudo_id)`) — paraphrase is not enough.
  - **Destination**: one `references/metrics/<slug>.md` file *per
    metric* (e.g. `references/metrics/daily_active_users.md`). The
    reference doc owns the SQL. Frontmatter: `type: Reference`, `tags:
    [metric]`, `resource` set to the page URL, plus the standard
    `title`/`description`/`timestamp`. Body: one-sentence definition,
    then a fenced SQL block with the formula, then a `# Citations`
    section.
  - Then add a `# Metrics` top-level section to each contributing
    table's primary doc (augmenting per the rules above) with one
    bullet per metric, e.g.
    `- [Daily active users](/references/metrics/daily_active_users.md) — DISTINCT user_pseudo_id per day.`
    Do **not** duplicate the SQL in the table doc; the reference owns it.
  - If the metric spans multiple tables, link it from every
    contributing table's `# Metrics` section.

- **Dimensions** (groupable / filterable attributes used in `GROUP BY`
  or `WHERE`, e.g. `event_name`, `device.category`, `traffic_source.medium`).
  Capture the column path, allowed values if enumerated, and a short
  semantic description.
  - **Destination**: the primary concept doc of the table that **owns
    the column**. Extend `# Schema` with the semantic description
    inline, OR add a `# Dimensions` sub-section listing dimension column
    paths and what each is good for.
  - For shared enum values that recur across tables (e.g. event-name
    catalogs), mint `references/<slug>.md` and cite from each table.

- **Join paths** (foreign-key relationships, recommended joins between
  tables in this bundle, e.g. *`events_.user_pseudo_id` ↔
  `users.user_pseudo_id`*). Capture the two sides and the **concrete
  `ON` clause**.
  - **Destination**: one `references/joins/<a>__<b>.md` file *per
    pair*, with the two table names sorted alphabetically and joined by
    a double underscore (e.g. `references/joins/events___users.md` for
    the `events_` ↔ `users` pair). One canonical file per pair,
    regardless of which side you came from. Frontmatter:
    `type: Reference`, `tags: [join]`, `resource` set to the page URL,
    plus the standard `title`/`description`/`timestamp`. Body: the
    `ON` clause as a fenced SQL block, then one sentence on when to use
    this join, then `# Citations`.
  - Then add a `# Joins` top-level section to **each** side's primary
    doc (augmenting per the rules above) with a one-line link to the
    reference, e.g.
    `- [users](/references/joins/events___users.md) — join on user_pseudo_id to attach user attributes to events.`
  - Do not invent join paths. Only capture joins explicitly named in
    documentation or example queries on the fetched page.

**These structured extractions bypass the four-gate reference test
above.** The gates exist to keep prose pages from becoming junk
references; metrics and joins are inherently concept-shaped and
inherently reusable, so they go straight into `references/metrics/` and
`references/joins/` without gate-checking. The four gates still apply
to *all other* `references/` mints.

If a page surfaces several of these at once (a typical "data model"
or "schema reference" page), make **multiple** `write_concept_doc`
calls — one per affected concept — rather than dumping everything into
one doc.

## Style and integrity

- Cite **only** URLs you actually fetched (or URLs already present in the
  doc you're refining). Do not invent URLs.
- Be concrete. Use concrete field names, concrete enum values, concrete
  example queries.
- Do not include preamble, apologies, or reasoning narration in document
  bodies. Bodies must be valid markdown ready for direct consumption.
- End your session with one short sentence summarizing what you did: how
  many pages you fetched, how many docs you updated, how many references
  you minted.


================================================
FILE: okf\src\reference_agent\runner.py
================================================
from __future__ import annotations

import json
import logging
import uuid
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

from google.adk.runners import Runner
from google.adk.sessions import InMemorySessionService
from google.genai import types

from reference_agent.agent import DEFAULT_MODEL, build_bq_agent, build_web_agent
from reference_agent.bundle.index import regenerate_indexes
from reference_agent.sources.base import ConceptRef, Source
from reference_agent.tools.context import (
    clear_web_state,
    set_context,
    set_web_state,
)

log = logging.getLogger(__name__)

_BQ_APP_NAME = "reference_agent_bq"
_WEB_APP_NAME = "reference_agent_web"
_USER_ID = "enricher"

_COMPACT_STR_LIMIT = 120
_COMPACT_TEXT_LIMIT = 200


def _summarize_value(value: Any, limit: int) -> str:
    if isinstance(value, str):
        return value if len(value) <= limit else f"<{len(value)} chars>"
    if isinstance(value, dict):
        if not value:
            return "{}"
        return f"{{{len(value)} keys}}"
    if isinstance(value, list):
        return f"[{len(value)} items]"
    if isinstance(value, (int, float, bool)) or value is None:
        return repr(value)
    return f"<{type(value).__name__}>"


def _compact_args(args: dict[str, Any] | None) -> str:
    if not args:
        return ""
    parts = [
        f"{k}={_summarize_value(v, _COMPACT_STR_LIMIT)}" for k, v in args.items()
    ]
    return ", ".join(parts)


def _compact_response(value: Any) -> str:
    if isinstance(value, dict):
        if not value:
            return "{}"
        # Surface useful scalar fields verbatim, summarize others.
        bits = []
        for k, v in value.items():
            bits.append(f"{k}={_summarize_value(v, _COMPACT_STR_LIMIT)}")
        return "{" + ", ".join(bits) + "}"
    return _summarize_value(value, _COMPACT_STR_LIMIT)


def _compact_text(text: str) -> str:
    one_line = " · ".join(s.strip() for s in text.splitlines() if s.strip())
    if len(one_line) <= _COMPACT_TEXT_LIMIT:
        return one_line
    return one_line[:_COMPACT_TEXT_LIMIT].rstrip() + " …"


def _full_json(value: Any) -> str:
    try:
        return json.dumps(value, indent=2, default=str, ensure_ascii=False)
    except Exception:
        return repr(value)


def _log_event_parts(event, prefix: str, *, verbose: bool) -> str | None:
    if not event.content or not event.content.parts:
        return None
    last_text: str | None = None
    for part in event.content.parts:
        fc = getattr(part, "function_call", None)
        fr = getattr(part, "function_response", None)
        text = getattr(part, "text", None)
        if fc and getattr(fc, "name", None):
            if verbose:
                log.info("[%s] → %s\n%s", prefix, fc.name, _full_json(fc.args or {}))
            else:
                log.info("[%s] → %s(%s)", prefix, fc.name, _compact_args(fc.args))
        elif fr and getattr(fr, "name", None):
            response = getattr(fr, "response", None)
            if verbose:
                log.info("[%s] ← %s\n%s", prefix, fr.name, _full_json(response))
            else:
                log.info("[%s] ← %s: %s", prefix, fr.name, _compact_response(response))
        elif text:
            stripped = text.strip()
            if not stripped:
                continue
            last_text = text
            if verbose:
                log.info("[%s] ✎ %s", prefix, stripped)
            else:
                log.info("[%s] ✎ %s", prefix, _compact_text(stripped))
    return last_text


def _build_bq_user_message(ref: ConceptRef) -> types.Content:
    text = (
        f"Enrich the concept with id: {ref.id_str}\n"
        f"OKF type: {ref.type}\n"
        f"Follow the standard workflow and write exactly one document for "
        f"this concept."
    )
    return types.Content(role="user", parts=[types.Part(text=text)])


def _build_web_user_message(
    seeds: list[str],
    max_pages: int,
    allowed_hosts: list[str],
    *,
    max_depth: int,
    allowed_path_prefixes: list[str],
    denied_path_substrings: list[str],
) -> types.Content:
    seed_lines = "\n".join(f"- {s}" for s in seeds)
    allowed_lines = ", ".join(sorted(allowed_hosts)) or "(any)"
    prefixes = ", ".join(allowed_path_prefixes) or "(any path)"
    denied = ", ".join(denied_path_substrings) or "(none)"
    text = (
        f"Ingest the following seed URLs and crawl outward as your judgment "
        f"directs.\n\n"
        f"Seed URLs:\n{seed_lines}\n\n"
        f"Hard limits enforced by the fetch_url tool — do not retry rejected "
        f"URLs:\n"
        f"- Max pages: {max_pages}\n"
        f"- Max hop depth from any seed: {max_depth}\n"
        f"- Allowed hosts: {allowed_lines}\n"
        f"- Allowed URL path prefixes: {prefixes}\n"
        f"- Denied URL path substrings: {denied}\n\n"
        f"Follow the web-ingestion workflow. For each fetched page, decide "
        f"whether it enriches an existing concept, deserves its own "
        f"`references/<slug>` doc, or should be skipped. Prefer skipping over "
        f"borderline fetches — the budget is small."
    )
    return types.Content(role="user", parts=[types.Part(text=text)])


class ReferenceRunner:
    def __init__(
        self,
        source: Source,
        bundle_root: Path,
        model: str = DEFAULT_MODEL,
        web_seeds: list[str] | None = None,
        web_max_pages: int = 100,
        web_allowed_hosts: set[str] | None = None,
        web_allowed_path_prefixes: list[str] | None = None,
        web_denied_path_substrings: list[str] | None = None,
        web_max_depth: int = 2,
        verbose: bool = False,
    ):
        self.source = source
        self.bundle_root = Path(bundle_root)
        self.model = model
        self.verbose = verbose
        self.bundle_root.mkdir(parents=True, exist_ok=True)
        set_context(self.source, self.bundle_root)

        self.web_seeds = list(web_seeds or [])
        self.web_max_pages = int(web_max_pages)
        self.web_allowed_path_prefixes = list(web_allowed_path_prefixes or [])
        self.web_denied_path_substrings = list(web_denied_path_substrings or [])
        self.web_max_depth = int(web_max_depth)
        if web_allowed_hosts is not None:
            self.web_allowed_hosts = set(web_allowed_hosts)
        else:
            self.web_allowed_hosts = {
                urlparse(s).netloc for s in self.web_seeds if urlparse(s).netloc
            }

        self._bq_agent = build_bq_agent(model=model)
        self._bq_session_service = InMemorySessionService()
        self._bq_runner = Runner(
            agent=self._bq_agent,
            app_name=_BQ_APP_NAME,
            session_service=self._bq_session_service,
        )

        self._web_runner: Runner | None = None
        if self.web_seeds:
            self._web_agent = build_web_agent(model=model)
            self._web_session_service = InMemorySessionService()
            self._web_runner = Runner(
                agent=self._web_agent,
                app_name=_WEB_APP_NAME,
                session_service=self._web_session_service,
            )

    def enrich_concept(self, ref: ConceptRef) -> None:
        session_id = f"enrich-{uuid.uuid4().hex[:12]}"
        self._bq_session_service.create_session_sync(
            app_name=_BQ_APP_NAME, user_id=_USER_ID, session_id=session_id
        )
        message = _build_bq_user_message(ref)
        for event in self._bq_runner.run(
            user_id=_USER_ID, session_id=session_id, new_message=message
        ):
            _log_event_parts(event, ref.id_str, verbose=self.verbose)

    def run_web_pass(self) -> None:
        if not self._web_runner or not self.web_seeds:
            return
        log.info(
            "Running web pass: %d seed(s), max_pages=%d, max_depth=%d, "
            "allowed_hosts=%s, allowed_path_prefixes=%s, "
            "denied_path_substrings=%s",
            len(self.web_seeds),
            self.web_max_pages,
            self.web_max_depth,
            sorted(self.web_allowed_hosts),
            self.web_allowed_path_prefixes,
            self.web_denied_path_substrings,
        )
        set_web_state(
            self.web_allowed_hosts,
            self.web_max_pages,
            seeds=self.web_seeds,
            allowed_path_prefixes=self.web_allowed_path_prefixes,
            denied_path_substrings=self.web_denied_path_substrings,
            max_depth=self.web_max_depth,
        )
        try:
            session_id = f"web-{uuid.uuid4().hex[:12]}"
            self._web_session_service.create_session_sync(
                app_name=_WEB_APP_NAME, user_id=_USER_ID, session_id=session_id
            )
            message = _build_web_user_message(
                self.web_seeds,
                self.web_max_pages,
                sorted(self.web_allowed_hosts),
                max_depth=self.web_max_depth,
                allowed_path_prefixes=self.web_allowed_path_prefixes,
                denied_path_substrings=self.web_denied_path_substrings,
            )
            for event in self._web_runner.run(
                user_id=_USER_ID, session_id=session_id, new_message=message
            ):
                _log_event_parts(event, "web", verbose=self.verbose)
        finally:
            clear_web_state()

    def enrich_all(self, only: list[tuple[str, ...]] | None = None) -> int:
        concepts = self.source.list_concepts()
        if only is not None:
            wanted = set(only)
            concepts = [c for c in concepts if c.id in wanted]
            missing = wanted - {c.id for c in concepts}
            if missing:
                raise ValueError(
                    f"Unknown concept(s): {sorted('/'.join(m) for m in missing)}"
                )

        count = 0
        for ref in concepts:
            log.info("Enriching %s (%s)", ref.id_str, ref.type)
            self.enrich_concept(ref)
            count += 1

        self.run_web_pass()

        log.info("Regenerating index.md files in %s", self.bundle_root)
        regenerate_indexes(self.bundle_root, model=self.model)
        return count


================================================
FILE: okf\src\reference_agent\sources\base.py
================================================
from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any


@dataclass(frozen=True)
class ConceptRef:
    id: tuple[str, ...]
    type: str
    resource: str | None = None
    hint: dict[str, Any] = field(default_factory=dict)

    @property
    def id_str(self) -> str:
        return "/".join(self.id)


class Source(ABC):
    name: str = ""

    @abstractmethod
    def list_concepts(self) -> list[ConceptRef]:
        ...

    @abstractmethod
    def read_concept(self, ref: ConceptRef) -> dict[str, Any]:
        ...

    def sample_rows(self, ref: ConceptRef, n: int = 5) -> list[dict[str, Any]] | None:
        return None

    def find(self, concept_id: tuple[str, ...]) -> ConceptRef | None:
        for ref in self.list_concepts():
            if ref.id == concept_id:
                return ref
        return None


================================================
FILE: okf\src\reference_agent\sources\bigquery.py
================================================
from __future__ import annotations

import re
from typing import Any

from google.cloud import bigquery

from reference_agent.sources.base import ConceptRef, Source

_SHARD_SUFFIX_RE = re.compile(r"^(?P<prefix>.+?_)(?P<shard>\d{6,8})$")


def _schema_to_dict(fields: list[bigquery.SchemaField]) -> list[dict[str, Any]]:
    out: list[dict[str, Any]] = []
    for f in fields:
        entry: dict[str, Any] = {
            "name": f.name,
            "type": f.field_type,
            "mode": f.mode,
        }
        if f.description:
            entry["description"] = f.description
        if f.fields:
            entry["fields"] = _schema_to_dict(list(f.fields))
        out.append(entry)
    return out


class BigQuerySource(Source):
    name = "bigquery"

    def __init__(self, dataset: str, billing_project: str | None = None):
        if "." not in dataset:
            raise ValueError(
                "dataset must be in 'project.dataset' form, "
                f"got {dataset!r}"
            )
        self.dataset_project, self.dataset_id = dataset.split(".", 1)
        self.client = bigquery.Client(project=billing_project)
        self._dataset_ref = bigquery.DatasetReference(
            self.dataset_project, self.dataset_id
        )
        self._concepts_cache: list[ConceptRef] | None = None
        self._family_index: dict[tuple[str, ...], list[str]] = {}

    def _dataset_resource_uri(self) -> str:
        return (
            f"https://bigquery.googleapis.com/v2/projects/"
            f"{self.dataset_project}/datasets/{self.dataset_id}"
        )

    def _table_resource_uri(self, table_id: str) -> str:
        return (
            f"https://bigquery.googleapis.com/v2/projects/"
            f"{self.dataset_project}/datasets/{self.dataset_id}/tables/{table_id}"
        )

    def list_concepts(self) -> list[ConceptRef]:
        if self._concepts_cache is not None:
            return self._concepts_cache

        concepts: list[ConceptRef] = []
        concepts.append(
            ConceptRef(
                id=("datasets", self.dataset_id),
                type="BigQuery Dataset",
                resource=self._dataset_resource_uri(),
                hint={
                    "dataset_project": self.dataset_project,
                    "dataset_id": self.dataset_id,
                },
            )
        )

        families: dict[str, list[str]] = {}
        singletons: list[str] = []
        for tbl in self.client.list_tables(self._dataset_ref):
            m = _SHARD_SUFFIX_RE.match(tbl.table_id)
            if m:
                families.setdefault(m.group("prefix"), []).append(tbl.table_id)
            else:
                singletons.append(tbl.table_id)

        for prefix, shards in sorted(families.items()):
            family_concept_id = ("tables", prefix)
            shards_sorted = sorted(shards)
            concepts.append(
                ConceptRef(
                    id=family_concept_id,
                    type="BigQuery Table",
                    resource=self._table_resource_uri(f"{prefix}*"),
                    hint={
                        "wildcard": True,
                        "family_prefix": prefix,
                        "shard_count": len(shards),
                        "first_shard": shards_sorted[0],
                        "last_shard": shards_sorted[-1],
                    },
                )
            )
            self._family_index[family_concept_id] = shards_sorted

        for table_id in sorted(singletons):
            concepts.append(
                ConceptRef(
                    id=("tables", table_id),
                    type="BigQuery Table",
                    resource=self._table_resource_uri(table_id),
                    hint={"wildcard": False, "table_id": table_id},
                )
            )

        self._concepts_cache = concepts
        return concepts

    def _representative_table_id(self, ref: ConceptRef) -> str:
        if ref.hint.get("wildcard"):
            return ref.hint["last_shard"]
        return ref.hint["table_id"]

    def read_concept(self, ref: ConceptRef) -> dict[str, Any]:
        if ref.type == "BigQuery Dataset":
            ds = self.client.get_dataset(self._dataset_ref)
            return {
                "dataset_project": self.dataset_project,
                "dataset_id": self.dataset_id,
                "friendly_name": ds.friendly_name,
                "description": ds.description,
                "location": ds.location,
                "labels": dict(ds.labels or {}),
                "created": ds.created.isoformat() if ds.created else None,
                "modified": ds.modified.isoformat() if ds.modified else None,
                "default_partition_expiration_ms": ds.default_partition_expiration_ms,
            }

        if ref.type == "BigQuery Table":
            self.list_concepts()
            table_id = self._representative_table_id(ref)
            tbl = self.client.get_table(self._dataset_ref.table(table_id))
            data: dict[str, Any] = {
                "dataset_project": self.dataset_project,
                "dataset_id": self.dataset_id,
                "representative_table_id": table_id,
                "wildcard": bool(ref.hint.get("wildcard")),
                "friendly_name": tbl.friendly_name,
                "description": tbl.description,
                "labels": dict(tbl.labels or {}),
                "num_rows": tbl.num_rows,
                "num_bytes": tbl.num_bytes,
                "created": tbl.created.isoformat() if tbl.created else None,
                "modified": tbl.modified.isoformat() if tbl.modified else None,
                "schema": _schema_to_dict(list(tbl.schema or [])),
            }
            if ref.hint.get("wildcard"):
                data["family_prefix"] = ref.hint["family_prefix"]
                data["shard_count"] = ref.hint["shard_count"]
                data["first_shard"] = ref.hint["first_shard"]
                data["last_shard"] = ref.hint["last_shard"]
            if tbl.time_partitioning:
                data["time_partitioning"] = {
                    "type": tbl.time_partitioning.type_,
                    "field": tbl.time_partitioning.field,
                    "expiration_ms": tbl.time_partitioning.expiration_ms,
                }
            if tbl.range_partitioning:
                rp = tbl.range_partitioning
                data["range_partitioning"] = {
                    "field": rp.field,
                    "range": {
                        "start": rp.range_.start,
                        "end": rp.range_.end,
                        "interval": rp.range_.interval,
                    },
                }
            if tbl.clustering_fields:
                data["clustering_fields"] = list(tbl.clustering_fields)
            return data

        raise ValueError(f"Unknown concept type: {ref.type}")

    def sample_rows(
        self, ref: ConceptRef, n: int = 5
    ) -> list[dict[str, Any]] | None:
        if ref.type != "BigQuery Table":
            return None
        self.list_concepts()
        table_id = self._representative_table_id(ref)
        table_ref = self._dataset_ref.table(table_id)
        try:
            tbl = self.client.get_table(table_ref)
        except Exception:
            return None
        table_type = (getattr(tbl, "table_type", None) or "TABLE").upper()
        try:
            if table_type == "TABLE":
                row_iter = self.client.list_rows(table_ref, max_results=n)
            else:
                # VIEW / MATERIALIZED_VIEW / EXTERNAL / SNAPSHOT — the
                # tabledata.list REST endpoint refuses non-base-tables, so
                # fall back to a small query that materializes the rows.
                sql = (
                    f"SELECT * FROM `{self.dataset_project}."
                    f"{self.dataset_id}.{table_id}` LIMIT {int(n)}"
                )
                row_iter = self.client.query(sql).result()
            return [dict(r.items()) for r in row_iter]
        except Exception:
            return None


================================================
FILE: okf\src\reference_agent\sources\__init__.py
================================================
from reference_agent.sources.base import ConceptRef, Source

__all__ = ["ConceptRef", "Source"]


================================================
FILE: okf\src\reference_agent\tools\bundle_tools.py
================================================
from __future__ import annotations

import re
from datetime import datetime, timezone
from typing import Any

from reference_agent.bundle.document import (
    REQUIRED_FRONTMATTER_KEYS,
    OKFDocument,
    OKFDocumentError,
)
from reference_agent.bundle.paths import concept_id_to_path, parse_concept_id
from reference_agent.tools.context import get_context, is_web_pass

_PREFERRED_KEY_ORDER = ("type", "resource", "title", "description", "tags", "timestamp")

_FIELD_NAME_RE = re.compile(r"`([A-Za-z_][A-Za-z0-9_.]*)`")


def _section_content_lines(body: str, heading: str) -> list[str]:
    """Return non-blank lines under a top-level `# heading` section."""
    in_section = False
    out: list[str] = []
    for line in body.splitlines():
        stripped = line.strip()
        if stripped.startswith("# "):
            in_section = stripped == heading
            continue
        if in_section and stripped:
            out.append(line)
    return out


def _schema_field_names(body: str) -> set[str]:
    names: set[str] = set()
    for line in _section_content_lines(body, "# Schema"):
        names.update(_FIELD_NAME_RE.findall(line))
    return names


def _citation_entry_count(body: str) -> int:
    return len(_section_content_lines(body, "# Citations"))


def _reorder_frontmatter(fm: dict[str, Any]) -> dict[str, Any]:
    ordered: dict[str, Any] = {}
    for key in _PREFERRED_KEY_ORDER:
        if key in fm:
            ordered[key] = fm[key]
    for key, value in fm.items():
        if key not in ordered:
            ordered[key] = value
    return ordered


def read_existing_doc(concept_id: str) -> dict[str, Any] | None:
    """Return the existing OKF document for this concept, if one is already on
    disk.

    Use this before writing to refine prior content instead of overwriting
    blindly. Returns null when no document exists yet. When a document exists,
    returns {'frontmatter': <object>, 'body': <markdown string>}.
    """
    ctx = get_context()
    cid = parse_concept_id(concept_id)
    path = concept_id_to_path(ctx.bundle_root, cid)
    if not path.exists():
        return None
    doc = OKFDocument.parse(path.read_text(encoding="utf-8"))
    return {"frontmatter": doc.frontmatter, "body": doc.body}


def write_concept_doc(
    concept_id: str,
    frontmatter: dict[str, Any],
    body: str,
) -> dict[str, Any]:
    """Write (or overwrite) the OKF markdown document for this concept.

    `frontmatter` must include at minimum: type, title, description, timestamp
    (ISO 8601). `resource` and `tags` are strongly recommended when applicable.
    The `body` should contain the prose description plus `# Schema`,
    `# Common query patterns`, and `# Citations` sections per the OKF
    convention.

    Returns {'path': <relative path written>, 'bytes': <int>}.
    """
    ctx = get_context()
    cid = parse_concept_id(concept_id)
    path = concept_id_to_path(ctx.bundle_root, cid)

    fm = dict(frontmatter)
    if not fm.get("timestamp"):
        fm["timestamp"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
    fm = _reorder_frontmatter(fm)

    doc = OKFDocument(frontmatter=fm, body=body or "")
    try:
        doc.validate()
    except OKFDocumentError as e:
        return {
            "error": (
                f"Refusing to write document with invalid frontmatter: {e}. "
                f"Required keys: {', '.join(REQUIRED_FRONTMATTER_KEYS)}. "
                f"Re-call write_concept_doc with the complete frontmatter dict."
            ),
            "concept_id": concept_id,
        }

    # Augmentation guard: during the web pass, refuse writes that shrink
    # an existing BigQuery Table doc's # Schema field set or # Citations
    # entry count. The BQ pass populates these from real metadata; the
    # web pass must augment, not replace.
    if is_web_pass() and path.exists():
        try:
            existing = OKFDocument.parse(path.read_text(encoding="utf-8"))
        except Exception:
            existing = None
        if existing is not None and existing.frontmatter.get("type") == "BigQuery Table":
            old_fields = _schema_field_names(existing.body)
            new_fields = _schema_field_names(body or "")
            missing = sorted(old_fields - new_fields)
            if missing:
                shown = ", ".join(f"`{m}`" for m in missing[:10])
                truncated = " (and more)" if len(missing) > 10 else ""
                return {
                    "error": (
                        f"Refusing to write: the existing # Schema section "
                        f"lists {len(old_fields)} field(s) populated from "
                        f"BigQuery metadata, but your new # Schema is "
                        f"missing {len(missing)} of them: {shown}"
                        f"{truncated}. Augment by adding to the existing "
                        f"schema, not replacing it. Re-call "
                        f"read_existing_doc to see the current schema, "
                        f"then re-call write_concept_doc with the full "
                        f"field list preserved."
                    ),
                    "concept_id": concept_id,
                }
            old_cites = _citation_entry_count(existing.body)
            new_cites = _citation_entry_count(body or "")
            if new_cites < old_cites:
                return {
                    "error": (
                        f"Refusing to write: the existing # Citations "
                        f"section had {old_cites} entries (including the "
                        f"BigQuery resource URL), but your new # Citations "
                        f"has only {new_cites}. Append your new citation "
                        f"rather than replacing the list. Re-call "
                        f"write_concept_doc with every existing entry "
                        f"preserved plus the new one."
                    ),
                    "concept_id": concept_id,
                }

    path.parent.mkdir(parents=True, exist_ok=True)
    text = doc.serialize()
    path.write_text(text, encoding="utf-8")
    return {
        "path": str(path.relative_to(ctx.bundle_root)),
        "bytes": len(text.encode("utf-8")),
    }


================================================
FILE: okf\src\reference_agent\tools\context.py
================================================
from __future__ import annotations

from dataclasses import dataclass, field
from pathlib import Path

from reference_agent.sources.base import Source


@dataclass
class ToolContext:
    source: Source
    bundle_root: Path


@dataclass
class WebState:
    allowed_hosts: set[str]
    max_pages: int
    allowed_path_prefixes: tuple[str, ...] = ()
    denied_path_substrings: tuple[str, ...] = ()
    max_depth: int = 2
    visited: set[str] = field(default_factory=set)
    fetched_count: int = 0
    url_depth: dict[str, int] = field(default_factory=dict)


_ctx: ToolContext | None = None
_web: WebState | None = None


def set_context(source: Source, bundle_root: Path) -> None:
    global _ctx
    _ctx = ToolContext(source=source, bundle_root=Path(bundle_root))


def get_context() -> ToolContext:
    if _ctx is None:
        raise RuntimeError(
            "Tool context not set. Call set_context() before invoking the agent."
        )
    return _ctx


def set_web_state(
    allowed_hosts: set[str],
    max_pages: int,
    *,
    seeds: list[str] | None = None,
    allowed_path_prefixes: list[str] | None = None,
    denied_path_substrings: list[str] | None = None,
    max_depth: int = 2,
) -> None:
    global _web
    _web = WebState(
        allowed_hosts=set(allowed_hosts),
        max_pages=int(max_pages),
        allowed_path_prefixes=tuple(allowed_path_prefixes or ()),
        denied_path_substrings=tuple(denied_path_substrings or ()),
        max_depth=int(max_depth),
    )
    for seed in seeds or ():
        _web.url_depth[seed] = 0


def get_web_state() -> WebState:
    if _web is None:
        raise RuntimeError(
            "Web state not set. Call set_web_state() before invoking the web agent."
        )
    return _web


def clear_web_state() -> None:
    global _web
    _web = None


def is_web_pass() -> bool:
    """True while the runner is executing the web-ingestion pass."""
    return _web is not None


================================================
FILE: okf\src\reference_agent\tools\source_tools.py
================================================
from __future__ import annotations

from typing import Any

from reference_agent.bundle.paths import parse_concept_id
from reference_agent.tools.context import get_context


def _ref_to_dict(ref) -> dict[str, Any]:
    return {
        "id": ref.id_str,
        "type": ref.type,
        "resource": ref.resource,
        "hint": dict(ref.hint),
    }


def list_concepts() -> list[dict[str, Any]]:
    """List every concept the active source advertises.

    Returns a list of objects with fields: `id` (slash-joined path used as the
    concept_id in other tools), `type` (OKF type, e.g. 'BigQuery Table'),
    `resource` (canonical URI of the underlying asset, if any), and `hint`
    (source-specific extra info such as wildcard flags or shard counts).
    """
    src = get_context().source
    return [_ref_to_dict(r) for r in src.list_concepts()]


def read_concept_raw(concept_id: str) -> dict[str, Any]:
    """Fetch raw structured metadata for a single concept from its source.

    `concept_id` is the slash-joined id returned by `list_concepts` (e.g.
    'tables/events_'). For BigQuery tables this includes schema (with nested
    RECORD fields), partitioning, clustering, row counts, and timestamps.
    """
    src = get_context().source
    cid = parse_concept_id(concept_id)
    ref = src.find(cid)
    if ref is None:
        raise ValueError(f"Unknown concept: {concept_id}")
    return src.read_concept(ref)


def sample_rows(concept_id: str, n: int = 5) -> dict[str, Any]:
    """Pull a small sample of rows from the underlying asset, if supported.

    Returns an object with `rows` (a list of stringified row dicts; empty if
    sampling is unsupported or fails) and `note` (a short human-readable
    explanation when rows could not be sampled).
    """
    src = get_context().source
    cid = parse_concept_id(concept_id)
    ref = src.find(cid)
    if ref is None:
        return {"rows": [], "note": f"Unknown concept: {concept_id}"}
    try:
        rows = src.sample_rows(ref, n=n)
    except Exception as e:
        return {"rows": [], "note": f"Sampling failed: {e}"}
    if rows is None:
        return {"rows": [], "note": "Sampling is not supported for this concept."}
    coerced = [{k: str(v) for k, v in row.items()} for row in rows]
    return {"rows": coerced, "note": ""}


================================================
FILE: okf\src\reference_agent\tools\web_tools.py
================================================
from __future__ import annotations

from typing import Any
from urllib.parse import urlparse

from reference_agent.tools.context import get_web_state
from reference_agent.web.fetcher import FetchError, fetch_and_parse


def fetch_url(url: str) -> dict[str, Any]:
    """Fetch a single web page and return its content as markdown plus its
    outbound links.

    The session-wide crawl budget (`max_pages`), the allowed-hosts filter,
    optional URL path-prefix allow-list, denied-substring blocklist, and a
    hop-depth cap measured from the seed URLs are all enforced inside this
    tool. When a fetch is rejected the return value contains an `error`
    field instead of page content. Treat that as a signal to stop or pick a
    different URL; do not retry the same URL.

    Successful return shape:
      {"url", "title", "markdown", "links",
       "fetched_count", "max_pages_budget", "depth", "max_depth"}

    Rejected return shape:
      {"error": "<reason>", "url": url,
       "fetched_count", "max_pages_budget"}
    """
    state = get_web_state()
    parsed = urlparse(url)

    def _reject(reason: str) -> dict[str, Any]:
        return {
            "error": reason,
            "url": url,
            "fetched_count": state.fetched_count,
            "max_pages_budget": state.max_pages,
        }

    if parsed.scheme not in ("http", "https"):
        return _reject(f"unsupported scheme: {parsed.scheme or '(none)'}")
    if not parsed.netloc:
        return _reject("missing host in URL")
    if state.allowed_hosts and parsed.netloc not in state.allowed_hosts:
        return _reject(
            f"host not in allowed list: {parsed.netloc} "
            f"(allowed: {sorted(state.allowed_hosts)})"
        )
    path = parsed.path or "/"
    if state.allowed_path_prefixes and not any(
        path.startswith(p) for p in state.allowed_path_prefixes
    ):
        return _reject(
            f"path not in allowed prefixes: {path} "
            f"(allowed: {list(state.allowed_path_prefixes)})"
        )
    for bad in state.denied_path_substrings:
        if bad and bad in path:
            return _reject(f"path matches denied substring: {bad!r}")
    if url in state.visited:
        return _reject("already fetched in this session")
    if state.fetched_count >= state.max_pages:
        return _reject("max_pages reached")

    depth = state.url_depth.get(url)
    if depth is None:
        # Unknown URL — treat as the agent typing it in directly, which is
        # only allowed for the initial seeds (already pre-registered at
        # depth 0). Anything else means the agent invented a URL not
        # surfaced via a parent page; reject so we don't lose depth tracking.
        return _reject(
            "URL not reachable from a seed within the crawl graph "
            "(was not returned as a link by any fetched page)"
        )
    if depth > state.max_depth:
        return _reject(
            f"depth {depth} exceeds max_depth {state.max_depth}"
        )

    state.visited.add(url)
    state.fetched_count += 1

    try:
        page = fetch_and_parse(url)
    except FetchError as e:
        return _reject(f"fetch failed: {e}")

    child_depth = depth + 1
    for link in page.links:
        state.url_depth.setdefault(link, child_depth)

    return {
        "url": page.url,
        "title": page.title,
        "markdown": page.markdown,
        "links": page.links,
        "fetched_count": state.fetched_count,
        "max_pages_budget": state.max_pages,
        "depth": depth,
        "max_depth": state.max_depth,
    }


================================================
FILE: okf\src\reference_agent\tools\__init__.py
================================================


================================================
FILE: okf\src\reference_agent\viewer\generator.py
================================================
from __future__ import annotations

import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from reference_agent.bundle.document import OKFDocument, OKFDocumentError

_INDEX_NAME = "index.md"
_LINK_RE = re.compile(r"\]\(([^)\s]+\.md)(?:#[A-Za-z0-9_\-]*)?\)")
_TYPE_PALETTE = {
    "BigQuery Dataset": "#8b5cf6",
    "BigQuery Table": "#3b82f6",
    "Reference": "#10b981",
}
_DEFAULT_NODE_COLOR = "#94a3b8"


@dataclass
class Concept:
    id: str
    type: str
    title: str
    description: str
    resource: str
    tags: list[str]
    body: str
    links_to: list[str] = field(default_factory=list)

    def to_node(self) -> dict[str, Any]:
        color = _TYPE_PALETTE.get(self.type, _DEFAULT_NODE_COLOR)
        return {
            "data": {
                "id": self.id,
                "label": self.title or self.id,
                "type": self.type,
                "description": self.description,
                "resource": self.resource,
                "tags": self.tags,
                "color": color,
                "size": 30 + min(60, len(self.body) // 200),
            }
        }


def _extract_links(body: str, doc_dir: Path, bundle_root: Path) -> list[str]:
    out: list[str] = []
    seen: set[str] = set()
    bundle_root_resolved = bundle_root.resolve()
    for m in _LINK_RE.finditer(body):
        target = m.group(1)
        if "://" in target or target.startswith("/"):
            continue
        try:
            resolved = (doc_dir / target).resolve().relative_to(bundle_root_resolved)
        except ValueError:
            continue
        rel = resolved.as_posix()
        if rel.endswith(".md"):
            rel = rel[:-3]
        if rel and rel not in seen:
            seen.add(rel)
            out.append(rel)
    return out


def _walk_concepts(bundle_root: Path) -> list[Concept]:
    concepts: list[Concept] = []
    for md_path in sorted(bundle_root.rglob("*.md")):
        if md_path.name == _INDEX_NAME:
            continue
        rel = md_path.relative_to(bundle_root).with_suffix("")
        concept_id = "/".join(rel.parts)
        try:
            doc = OKFDocument.parse(md_path.read_text(encoding="utf-8"))
        except OKFDocumentError:
            continue
        fm = doc.frontmatter or {}
        tags = fm.get("tags") or []
        if not isinstance(tags, list):
            tags = [str(tags)]
        concept = Concept(
            id=concept_id,
            type=str(fm.get("type") or "Unknown"),
            title=str(fm.get("title") or concept_id),
            description=str(fm.get("description") or ""),
            resource=str(fm.get("resource") or ""),
            tags=[str(t) for t in tags],
            body=doc.body or "",
            links_to=_extract_links(doc.body or "", md_path.parent, bundle_root),
        )
        concepts.append(concept)
    return concepts


def _build_graph(concepts: list[Concept]) -> dict[str, Any]:
    ids = {c.id for c in concepts}
    nodes = [c.to_node() for c in concepts]
    edges: list[dict[str, Any]] = []
    seen_edges: set[tuple[str, str]] = set()
    for c in concepts:
        for target in c.links_to:
            if target == c.id or target not in ids:
                continue
            key = (c.id, target)
            if key in seen_edges:
                continue
            seen_edges.add(key)
            edges.append({
                "data": {
                    "id": f"{c.id}__{target}",
                    "source": c.id,
                    "target": target,
                }
            })
    bodies = {c.id: c.body for c in concepts}
    types = sorted({c.type for c in concepts})
    return {
        "nodes": nodes,
        "edges": edges,
        "bodies": bodies,
        "types": types,
        "palette": _TYPE_PALETTE,
    }


def _load_template() -> str:
    template_path = Path(__file__).parent / "templates" / "viz.html"
    return template_path.read_text(encoding="utf-8")


def _load_asset(name: str) -> str:
    asset_path = Path(__file__).parent / "static" / name
    return asset_path.read_text(encoding="utf-8")


def generate_visualization(
    bundle_root: Path,
    out_path: Path,
    *,
    bundle_name: str | None = None,
) -> dict[str, int]:
    """Walk a bundle and write a single self-contained HTML visualization.

    Returns counts: {'concepts': N, 'edges': M, 'bytes': K}.
    """
    bundle_root = Path(bundle_root)
    out_path = Path(out_path)
    if not bundle_root.is_dir():
        raise FileNotFoundError(f"Bundle directory not found: {bundle_root}")

    concepts = _walk_concepts(bundle_root)
    graph = _build_graph(concepts)
    template = _load_template()
    css = _load_asset("viz.css")
    js = _load_asset("viz.js")
    name = bundle_name or bundle_root.resolve().name

    html = (
        template
        .replace("/*__VIZ_CSS__*/", css)
        .replace("/*__VIZ_JS__*/", js)
        .replace("__BUNDLE_NAME__", json.dumps(name))
        .replace("__BUNDLE_DATA__", json.dumps(graph))
    )
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(html, encoding="utf-8")

    return {
        "concepts": len(concepts),
        "edges": len(graph["edges"]),
        "bytes": len(html.encode("utf-8")),
    }


================================================
FILE: okf\src\reference_agent\viewer\static\viz.css
================================================
* { box-sizing: border-box; }
body {
  margin: 0;
  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
  font-size: 14px;
  color: #0f172a;
  background: #f8fafc;
  display: flex;
  flex-direction: column;
  height: 100vh;
}
header {
  display: flex;
  align-items: center;
  justify-content: space-between;
  padding: 10px 16px;
  background: #fff;
  border-bottom: 1px solid #e2e8f0;
  flex-shrink: 0;
}
.title strong { font-size: 16px; margin-right: 8px; }
.muted { color: #64748b; font-size: 12px; }
.controls { display: flex; gap: 8px; }
.controls input, .controls select, .controls button {
  font-size: 13px;
  padding: 5px 8px;
  border: 1px solid #cbd5e1;
  border-radius: 4px;
  background: #fff;
}
.controls input { width: 220px; }
.controls button { cursor: pointer; background: #f1f5f9; }
.controls button:hover { background: #e2e8f0; }

main {
  display: flex;
  flex: 1;
  min-height: 0;
}
#graph {
  flex: 1 1 60%;
  background: #fff;
  border-right: 1px solid #e2e8f0;
  min-width: 0;
  position: relative;
}
#detail {
  flex: 0 0 40%;
  overflow-y: auto;
  padding: 18px 22px;
  background: #fff;
}
#detail-empty {
  text-align: center;
  margin-top: 40px;
}

.detail-header { margin-bottom: 12px; }
.detail-header h1 {
  font-size: 18px;
  margin: 4px 0 2px;
  font-weight: 600;
}
.type-chip {
  display: inline-block;
  padding: 2px 8px;
  border-radius: 10px;
  font-size: 11px;
  font-weight: 600;
  color: #fff;
  background: #94a3b8;
  text-transform: uppercase;
  letter-spacing: 0.5px;
}
dl.frontmatter {
  display: grid;
  grid-template-columns: 90px 1fr;
  row-gap: 4px;
  column-gap: 12px;
  margin: 8px 0 12px;
  font-size: 13px;
}
dl.frontmatter dt {
  color: #64748b;
  font-weight: 500;
}
dl.frontmatter dd { margin: 0; }
dl.frontmatter a { color: #2563eb; word-break: break-all; }

.tag {
  display: inline-block;
  padding: 1px 6px;
  margin: 0 4px 2px 0;
  border-radius: 4px;
  background: #f1f5f9;
  color: #475569;
  font-size: 11px;
}

hr { border: none; border-top: 1px solid #e2e8f0; margin: 14px 0; }

#detail-body { font-size: 13px; line-height: 1.55; }
#detail-body h1 {
  font-size: 16px; margin: 18px 0 6px;
  padding-bottom: 4px; border-bottom: 1px solid #e2e8f0;
}
#detail-body h2 { font-size: 14px; margin: 14px 0 4px; }
#detail-body h3 { font-size: 13px; margin: 12px 0 4px; }
#detail-body p { margin: 6px 0; }
#detail-body code {
  background: #f1f5f9;
  padding: 1px 4px;
  border-radius: 3px;
  font-size: 12px;
  font-family: ui-monospace, "SF Mono", Consolas, monospace;
}
#detail-body pre {
  background: #0f172a;
  color: #e2e8f0;
  padding: 10px 12px;
  border-radius: 6px;
  overflow-x: auto;
  font-size: 12px;
}
#detail-body pre code { background: transparent; color: inherit; padding: 0; }
#detail-body ul, #detail-body ol { padding-left: 22px; margin: 6px 0; }
#detail-body li { margin: 2px 0; }
#detail-body table { border-collapse: collapse; margin: 8px 0; }
#detail-body th, #detail-body td {
  border: 1px solid #e2e8f0; padding: 4px 8px; font-size: 12px;
}
#detail-body a.internal { color: #2563eb; cursor: pointer; }
#detail-body a.external { color: #2563eb; }

#detail-backlinks { margin-top: 18px; }
#detail-backlinks h2 { font-size: 13px; color: #64748b; margin-bottom: 6px; }
#detail-backlinks ul { padding-left: 18px; }
#detail-backlinks a { color: #2563eb; cursor: pointer; }


================================================
FILE: okf\src\reference_agent\viewer\static\viz.js
================================================
(function () {
  const bundle = window.BUNDLE;
  const bundleName = window.BUNDLE_NAME;
  document.title = `${bundleName} — OKF Viewer`;
  document.getElementById("bundle-name").textContent = bundleName;

  // Populate type filter
  const typeSelect = document.getElementById("filter-type");
  for (const t of bundle.types) {
    const opt = document.createElement("option");
    opt.value = t;
    opt.textContent = t;
    typeSelect.appendChild(opt);
  }

  // Build reverse-link index for backlinks
  const backlinks = {};
  for (const edge of bundle.edges) {
    const { source, target } = edge.data;
    (backlinks[target] ||= []).push(source);
  }

  // Look up node label/type by id
  const nodeIndex = {};
  for (const n of bundle.nodes) nodeIndex[n.data.id] = n.data;

  const cy = cytoscape({
    container: document.getElementById("graph"),
    elements: [...bundle.nodes, ...bundle.edges],
    style: [
      {
        selector: "node",
        style: {
          "background-color": "data(color)",
          "label": "data(label)",
          "color": "#0f172a",
          "font-size": 11,
          "text-valign": "bottom",
          "text-margin-y": 4,
          "text-wrap": "wrap",
          "text-max-width": 120,
          "width": "data(size)",
          "height": "data(size)",
          "border-width": 1,
          "border-color": "#0f172a",
        },
      },
      {
        selector: "node:selected",
        style: {
          "border-width": 3,
          "border-color": "#f59e0b",
        },
      },
      {
        selector: "edge",
        style: {
          "width": 1.5,
          "line-color": "#cbd5e1",
          "target-arrow-color": "#cbd5e1",
          "target-arrow-shape": "triangle",
          "curve-style": "bezier",
          "arrow-scale": 0.9,
        },
      },
      {
        selector: "edge:selected",
        style: {
          "line-color": "#f59e0b",
          "target-arrow-color": "#f59e0b",
          "width": 2.5,
        },
      },
      {
        selector: ".dim",
        style: { "opacity": 0.15 },
      },
    ],
    layout: { name: "cose", animate: false, padding: 30 },
    wheelSensitivity: 0.2,
  });

  cy.on("tap", "node", (evt) => showDetail(evt.target.id()));
  cy.on("tap", (evt) => {
    if (evt.target === cy) clearSelection();
  });

  document.getElementById("layout").addEventListener("change", (e) => {
    cy.layout({ name: e.target.value, animate: false, padding: 30 }).run();
  });

  document.getElementById("reset").addEventListener("click", () => {
    cy.fit(null, 30);
    clearSelection();
  });

  document.getElementById("search").addEventListener("input", (e) => {
    const q = e.target.value.trim().toLowerCase();
    if (!q) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      const d = n.data();
      const hay =
        (d.label || "").toLowerCase() + " " +
        d.id.toLowerCase() + " " +
        (d.tags || []).join(" ").toLowerCase();
      n.toggleClass("dim", !hay.includes(q));
    });
    cy.edges().forEach((edge) => {
      const src = edge.source();
      const tgt = edge.target();
      edge.toggleClass("dim", src.hasClass("dim") || tgt.hasClass("dim"));
    });
  });

  document.getElementById("filter-type").addEventListener("change", (e) => {
    const t = e.target.value;
    if (!t) {
      cy.elements().removeClass("dim");
      return;
    }
    cy.nodes().forEach((n) => {
      n.toggleClass("dim", n.data("type") !== t);
    });
    cy.edges().forEach((edge) => {
      edge.toggleClass("dim", edge.source().hasClass("dim") || edge.target().hasClass("dim"));
    });
  });

  function clearSelection() {
    cy.elements().unselect();
    document.getElementById("detail-empty").hidden = false;
    document.getElementById("detail-content").hidden = true;
  }

  function showDetail(conceptId) {
    const data = nodeIndex[conceptId];
    if (!data) return;
    cy.elements().unselect();
    const node = cy.getElementById(conceptId);
    if (node) node.select();

    document.getElementById("detail-empty").hidden = true;
    const content = document.getElementById("detail-content");
    content.hidden = false;

    const chip = document.getElementById("detail-type");
    chip.textContent = data.type;
    chip.style.background = data.color;

    document.getElementById("detail-title").textContent = data.label;
    document.getElementById("detail-id").textContent = conceptId;
    document.getElementById("detail-description").textContent = data.description || "—";

    const resourceEl = document.getElementById("detail-resource");
    resourceEl.innerHTML = "";
    if (data.resource) {
      const a = document.createElement("a");
      a.href = data.resource;
      a.textContent = data.resource;
      a.target = "_blank";
      a.rel = "noopener";
      a.className = "external";
      resourceEl.appendChild(a);
    } else {
      resourceEl.textContent = "—";
    }

    const tagsEl = document.getElementById("detail-tags");
    tagsEl.innerHTML = "";
    if (data.tags && data.tags.length) {
      for (const t of data.tags) {
        const span = document.createElement("span");
        span.className = "tag";
        span.textContent = t;
        tagsEl.appendChild(span);
      }
    } else {
      tagsEl.textContent = "—";
    }

    const body = bundle.bodies[conceptId] || "";
    const html = marked.parse(body, { breaks: false, gfm: true });
    const bodyEl = document.getElementById("detail-body");
    bodyEl.innerHTML = html;
    rewriteInternalLinks(bodyEl);

    const bl = backlinks[conceptId] || [];
    const blSection = document.getElementById("detail-backlinks");
    const blList = document.getElementById("backlinks-list");
    blList.innerHTML = "";
    if (bl.length) {
      blSection.hidden = false;
      for (const src of bl) {
        const li = document.createElement("li");
        const a = document.createElement("a");
        a.textContent = nodeIndex[src]?.label || src;
        a.dataset.target = src;
        a.addEventListener("click", () => showDetail(src));
        li.appendChild(a);
        const muted = document.createElement("span");
        muted.className = "muted";
        muted.textContent = ` (${src})`;
        li.appendChild(muted);
        blList.appendChild(li);
      }
    } else {
      blSection.hidden = true;
    }

    cy.animate({ center: { eles: node }, zoom: Math.max(cy.zoom(), 1.0) }, { duration: 200 });
  }

  function rewriteInternalLinks(root) {
    root.querySelectorAll("a[href]").forEach((a) => {
      const href = a.getAttribute("href");
      if (!href) return;
      if (href.startsWith("/") && href.endsWith(".md")) {
        const target = href.slice(1, -3);
        if (nodeIndex[target]) {
          a.className = "internal";
          a.setAttribute("href", "javascript:void(0)");
          a.addEventListener("click", (e) => {
            e.preventDefault();
            showDetail(target);
          });
          return;
        }
      }
      a.className = "external";
      a.setAttribute("target", "_blank");
      a.setAttribute("rel", "noopener");
    });
  }

  // Auto-show the first node (a dataset if available, else first concept)
  const initial =
    bundle.nodes.find((n) => n.data.type === "BigQuery Dataset") ||
    bundle.nodes[0];
  if (initial) showDetail(initial.data.id);
})();


================================================
FILE: okf\src\reference_agent\viewer\templates\viz.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>OKF Bundle Viewer</title>
<script src="https://cdn.jsdelivr.net/npm/cytoscape@3.28.1/dist/cytoscape.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/marked@12.0.0/marked.min.js"></script>
<style>
/*__VIZ_CSS__*/
</style>
</head>
<body>
<header>
  <div class="title">
    <strong id="bundle-name"></strong>
    <span class="muted">OKF bundle</span>
  </div>
  <div class="controls">
    <input id="search" type="search" placeholder="Search title / id / tag">
    <select id="filter-type">
      <option value="">All types</option>
    </select>
    <select id="layout">
      <option value="cose">cose (force)</option>
      <option value="concentric">concentric</option>
      <option value="breadthfirst">breadth-first</option>
      <option value="circle">circle</option>
      <option value="grid">grid</option>
    </select>
    <button id="reset">Reset view</button>
  </div>
</header>

<main>
  <section id="graph"></section>
  <section id="detail">
    <div id="detail-empty" class="muted">Click a node to see its details.</div>
    <article id="detail-content" hidden>
      <header class="detail-header">
        <span class="type-chip" id="detail-type"></span>
        <h1 id="detail-title"></h1>
        <div class="muted" id="detail-id"></div>
      </header>
      <dl class="frontmatter">
        <dt>Description</dt><dd id="detail-description"></dd>
        <dt>Resource</dt><dd id="detail-resource"></dd>
        <dt>Tags</dt><dd id="detail-tags"></dd>
      </dl>
      <hr>
      <div id="detail-body"></div>
      <section id="detail-backlinks" hidden>
        <h2>Cited by</h2>
        <ul id="backlinks-list"></ul>
      </section>
    </article>
  </section>
</main>

<script>
window.BUNDLE_NAME = __BUNDLE_NAME__;
window.BUNDLE = __BUNDLE_DATA__;
</script>
<script>
/*__VIZ_JS__*/
</script>
</body>
</html>


================================================
FILE: okf\src\reference_agent\viewer\__init__.py
================================================
from reference_agent.viewer.generator import generate_visualization

__all__ = ["generate_visualization"]


================================================
FILE: okf\src\reference_agent\web\fetcher.py
================================================
from __future__ import annotations

import re
from dataclasses import dataclass
from urllib.parse import urldefrag, urljoin, urlparse
from urllib.request import Request, urlopen

from markdownify import markdownify

_USER_AGENT = "okf-reference-agent/0.1 (+https://github.com/amirhormati/open-knowledge-format)"
_MAX_MARKDOWN_BYTES = 40 * 1024

_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
_HREF_RE = re.compile(r"""href\s*=\s*["']([^"'#\s]+)["']""", re.IGNORECASE)


class FetchError(Exception):
    pass


@dataclass(frozen=True)
class Page:
    url: str
    title: str | None
    markdown: str
    links: list[str]


def _extract_title(html: str) -> str | None:
    m = _TITLE_RE.search(html)
    if not m:
        return None
    raw = re.sub(r"\s+", " ", m.group(1)).strip()
    return raw or None


def _extract_links(html: str, base_url: str) -> list[str]:
    seen: set[str] = set()
    out: list[str] = []
    for match in _HREF_RE.finditer(html):
        href = match.group(1).strip()
        if not href:
            continue
        scheme = urlparse(href).scheme.lower()
        if scheme and scheme not in ("http", "https", ""):
            continue
        absolute, _ = urldefrag(urljoin(base_url, href))
        if absolute in seen:
            continue
        seen.add(absolute)
        out.append(absolute)
    return out


def _truncate(text: str, max_bytes: int) -> str:
    encoded = text.encode("utf-8", errors="replace")
    if len(encoded) <= max_bytes:
        return text
    return encoded[:max_bytes].decode("utf-8", errors="ignore") + "\n\n[...truncated...]"


def fetch_and_parse(url: str, *, timeout: float = 10.0) -> Page:
    req = Request(url, headers={"User-Agent": _USER_AGENT, "Accept": "text/html,*/*;q=0.5"})
    try:
        with urlopen(req, timeout=timeout) as resp:
            content_type = resp.headers.get("Content-Type", "")
            final_url = resp.geturl() or url
            body_bytes = resp.read()
    except Exception as e:
        raise FetchError(str(e)) from e

    if "html" not in content_type.lower():
        raise FetchError(f"non-HTML content-type: {content_type or 'unknown'}")

    charset = "utf-8"
    if "charset=" in content_type.lower():
        charset = content_type.lower().split("charset=", 1)[1].split(";", 1)[0].strip() or "utf-8"
    try:
        html = body_bytes.decode(charset, errors="replace")
    except LookupError:
        html = body_bytes.decode("utf-8", errors="replace")

    title = _extract_title(html)
    links = _extract_links(html, final_url)
    markdown = markdownify(html, heading_style="ATX")
    markdown = _truncate(markdown, _MAX_MARKDOWN_BYTES)

    return Page(url=final_url, title=title, markdown=markdown, links=links)


================================================
FILE: okf\src\reference_agent\web\__init__.py
================================================


================================================
FILE: okf\src\reference_agent\__init__.py
================================================
__version__ = "0.1.0"


================================================
FILE: okf\src\reference_agent\__main__.py
================================================
from reference_agent.cli import main

if __name__ == "__main__":
    raise SystemExit(main())


================================================
FILE: okf\tests\test_bigquery_source.py
================================================
from __future__ import annotations

from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import MagicMock, patch


def _table_ref(table_id: str) -> SimpleNamespace:
    return SimpleNamespace(table_id=table_id)


def _schema_field(
    name: str, type_: str, mode: str = "NULLABLE", description: str | None = None,
    fields=()
):
    return SimpleNamespace(
        name=name,
        field_type=type_,
        mode=mode,
        description=description,
        fields=tuple(fields),
    )


def _build_mock_client(table_ids, table_objects_by_id):
    client = MagicMock()
    client.list_tables.return_value = [_table_ref(t) for t in table_ids]

    def get_table(table_ref):
        return table_objects_by_id[table_ref.table_id]

    client.get_table.side_effect = get_table
    return client


@patch("reference_agent.sources.bigquery.bigquery.Client")
def test_wildcard_sharded_tables_collapse_to_one_concept(client_cls):
    table_ids = [
        "events_20210101",
        "events_20210102",
        "events_20210103",
        "users",
    ]
    table_objects = {
        tid: SimpleNamespace(
            friendly_name=None,
            description="GA4 events sample." if tid.startswith("events_") else "Users dim.",
            labels={},
            num_rows=10,
            num_bytes=1024,
            created=datetime(2026, 5, 1, tzinfo=timezone.utc),
            modified=datetime(2026, 5, 1, tzinfo=timezone.utc),
            schema=[_schema_field("event_date", "STRING")],
            time_partitioning=None,
            range_partitioning=None,
            clustering_fields=None,
        )
        for tid in table_ids
    }
    client_cls.return_value = _build_mock_client(table_ids, table_objects)

    from reference_agent.sources.bigquery import BigQuerySource

    src = BigQuerySource(dataset="proj.dset")
    concepts = src.list_concepts()
    ids = [c.id for c in concepts]

    assert ("datasets", "dset") in ids
    assert ("tables", "events_") in ids
    assert ("tables", "users") in ids
    assert not any(c.id == ("tables", "events_20210101") for c in concepts)

    family = next(c for c in concepts if c.id == ("tables", "events_"))
    assert family.type == "BigQuery Table"
    assert family.hint["wildcard"] is True
    assert family.hint["shard_count"] == 3
    assert family.hint["last_shard"] == "events_20210103"


@patch("reference_agent.sources.bigquery.bigquery.Client")
def test_sample_rows_uses_list_rows_for_table(client_cls):
    table_objects = {
        "users": SimpleNamespace(table_type="TABLE"),
    }
    client = _build_mock_client(["users"], table_objects)
    client.list_rows.return_value = [
        SimpleNamespace(items=lambda: [("id", 1), ("name", "alice")]),
    ]
    client_cls.return_value = client

    from reference_agent.sources.bigquery import BigQuerySource

    src = BigQuerySource(dataset="proj.dset")
    ref = next(c for c in src.list_concepts() if c.id == ("tables", "users"))
    rows = src.sample_rows(ref, n=3)

    assert rows == [{"id": 1, "name": "alice"}]
    client.list_rows.assert_called_once()
    client.query.assert_not_called()


@patch("reference_agent.sources.bigquery.bigquery.Client")
def test_sample_rows_falls_back_to_query_for_view(client_cls):
    table_objects = {
        "user_summary": SimpleNamespace(table_type="VIEW"),
    }
    client = _build_mock_client(["user_summary"], table_objects)
    query_job = MagicMock()
    query_job.result.return_value = [
        SimpleNamespace(items=lambda: [("user_id", 42), ("orders", 7)]),
    ]
    client.query.return_value = query_job
    client_cls.return_value = client

    from reference_agent.sources.bigquery import BigQuerySource

    src = BigQuerySource(dataset="proj.dset")
    ref = next(c for c in src.list_concepts() if c.id == ("tables", "user_summary"))
    rows = src.sample_rows(ref, n=4)

    assert rows == [{"user_id": 42, "orders": 7}]
    client.list_rows.assert_not_called()
    client.query.assert_called_once()
    sql = client.query.call_args.args[0]
    assert "`proj.dset.user_summary`" in sql
    assert "LIMIT 4" in sql


@patch("reference_agent.sources.bigquery.bigquery.Client")
def test_read_concept_returns_schema_and_partitioning(client_cls):
    table_ids = ["events_20210103"]
    inner = [
        _schema_field("key", "STRING"),
        _schema_field("value", "RECORD", fields=[_schema_field("string_value", "STRING")]),
    ]
    schema = [
        _schema_field("event_date", "STRING"),
        _schema_field("event_params", "RECORD", mode="REPEATED", fields=inner),
    ]
    time_part = SimpleNamespace(type_="DAY", field=None, expiration_ms=None)
    table_objects = {
        "events_20210103": SimpleNamespace(
            friendly_name="Events",
            description="GA4 events table.",
            labels={"team": "ga4"},
            num_rows=100,
            num_bytes=2048,
            created=datetime(2026, 5, 1, tzinfo=timezone.utc),
            modified=datetime(2026, 5, 2, tzinfo=timezone.utc),
            schema=schema,
            time_partitioning=time_part,
            range_partitioning=None,
            clustering_fields=["user_pseudo_id"],
        )
    }
    client_cls.return_value = _build_mock_client(table_ids, table_objects)

    from reference_agent.sources.bigquery import BigQuerySource

    src = BigQuerySource(dataset="proj.dset")
    family = next(c for c in src.list_concepts() if c.id == ("tables", "events_"))

    data = src.read_concept(family)
    assert data["description"] == "GA4 events table."
    assert data["labels"] == {"team": "ga4"}
    assert data["schema"][0]["name"] == "event_date"
    nested = data["schema"][1]
    assert nested["name"] == "event_params"
    assert nested["mode"] == "REPEATED"
    assert nested["fields"][0]["name"] == "key"
    assert nested["fields"][1]["fields"][0]["name"] == "string_value"
    assert data["time_partitioning"]["type"] == "DAY"
    assert data["clustering_fields"] == ["user_pseudo_id"]
    assert data["wildcard"] is True
    assert data["family_prefix"] == "events_"
    assert data["last_shard"] == "events_20210103"


================================================
FILE: okf\tests\test_bundle_tools.py
================================================
from __future__ import annotations

from pathlib import Path
from unittest.mock import MagicMock

import pytest

from reference_agent.tools.bundle_tools import write_concept_doc
from reference_agent.tools.context import (
    clear_web_state,
    set_context,
    set_web_state,
)


@pytest.fixture(autouse=True)
def _cleanup():
    yield
    clear_web_state()


def _set_ctx(tmp_path: Path) -> None:
    src = MagicMock()
    set_context(src, tmp_path)


def _good_frontmatter(**overrides):
    fm = {
        "type": "BigQuery Table",
        "title": "Users",
        "description": "A table of users.",
        "resource": "https://bigquery.googleapis.com/v2/projects/p/datasets/d/tables/users",
        "tags": ["users"],
    }
    fm.update(overrides)
    return fm


def _bq_body(fields: list[str], citations: list[str]) -> str:
    schema_lines = "\n".join(f"- `{f}` STRING: desc" for f in fields)
    cite_lines = "\n".join(citations)
    return f"Prose.\n\n# Schema\n{schema_lines}\n\n# Citations\n{cite_lines}\n"


def test_write_succeeds_when_no_existing_doc(tmp_path):
    _set_ctx(tmp_path)
    set_web_state(allowed_hosts=set(), max_pages=1)
    result = write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name"], ["[1] [BQ](https://bq)"]),
    )
    assert "error" not in result
    assert (tmp_path / "tables" / "users.md").exists()


def test_web_pass_rejects_schema_shrinkage(tmp_path):
    _set_ctx(tmp_path)
    # Simulate the BQ pass having already written the doc.
    write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name", "email", "created_at"], ["[1] [BQ](https://bq)"]),
    )
    # Now enter the web pass.
    set_web_state(allowed_hosts=set(), max_pages=1)
    result = write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name"], ["[1] [BQ](https://bq)", "[2] [Web](https://web)"]),
    )
    assert "error" in result
    assert "missing 2" in result["error"]
    assert "`email`" in result["error"]
    assert "`created_at`" in result["error"]


def test_web_pass_rejects_citation_shrinkage(tmp_path):
    _set_ctx(tmp_path)
    write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id"], ["[1] [BQ](https://bq)", "[2] [Other](https://other)"]),
    )
    set_web_state(allowed_hosts=set(), max_pages=1)
    result = write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id"], ["[1] [Web](https://web)"]),
    )
    assert "error" in result
    assert "had 2 entries" in result["error"]


def test_web_pass_allows_augmentation_with_new_section(tmp_path):
    _set_ctx(tmp_path)
    write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name"], ["[1] [BQ](https://bq)"]),
    )
    set_web_state(allowed_hosts=set(), max_pages=1)
    augmented = (
        "Prose.\n\n# Schema\n- `id` STRING: desc\n- `name` STRING: desc\n\n"
        "# Metrics\n- [DAU](/references/metrics/dau.md) — count distinct id\n\n"
        "# Citations\n[1] [BQ](https://bq)\n[2] [Web](https://web)\n"
    )
    result = write_concept_doc("tables/users", _good_frontmatter(), augmented)
    assert "error" not in result


def test_bq_pass_can_shrink_schema_when_no_web_state(tmp_path):
    _set_ctx(tmp_path)
    # Initial doc with three fields.
    write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name", "legacy_col"], ["[1] [BQ](https://bq)"]),
    )
    # BQ pass re-runs (no web state) and the table evolved — legacy_col gone.
    result = write_concept_doc(
        "tables/users",
        _good_frontmatter(),
        _bq_body(["id", "name"], ["[1] [BQ](https://bq)"]),
    )
    assert "error" not in result


def test_web_pass_skips_guard_for_non_bigquery_table_types(tmp_path):
    _set_ctx(tmp_path)
    # An existing reference doc with two backtick-quoted things.
    ref_body = (
        "Prose.\n\n# Definition\nUses `field_a` and `field_b`.\n\n"
        "# Citations\n[1] [Src](https://src)\n"
    )
    write_concept_doc(
        "references/foo",
        _good_frontmatter(type="Reference", title="Foo"),
        ref_body,
    )
    set_web_state(allowed_hosts=set(), max_pages=1)
    # Drop both backticked identifiers — guard should not fire because
    # the existing doc is not type=BigQuery Table.
    result = write_concept_doc(
        "references/foo",
        _good_frontmatter(type="Reference", title="Foo"),
        "Different prose.\n\n# Definition\nNo more identifiers.\n\n"
        "# Citations\n[1] [Src](https://src)\n",
    )
    assert "error" not in result


================================================
FILE: okf\tests\test_document.py
================================================
from __future__ import annotations

import pytest

from reference_agent.bundle.document import OKFDocument, OKFDocumentError


def test_roundtrip_preserves_frontmatter_and_body():
    src = (
        "---\n"
        "type: BigQuery Table\n"
        "title: Sample\n"
        "description: A sample table.\n"
        "tags: [a, b]\n"
        "timestamp: 2026-05-27T00:00:00+00:00\n"
        "---\n"
        "\n"
        "# Sample\n"
        "\n"
        "Body text.\n"
    )
    doc = OKFDocument.parse(src)
    assert doc.frontmatter["type"] == "BigQuery Table"
    assert doc.frontmatter["tags"] == ["a", "b"]
    assert doc.body.startswith("# Sample")

    serialized = doc.serialize()
    reparsed = OKFDocument.parse(serialized)
    assert reparsed.frontmatter == doc.frontmatter
    assert reparsed.body.strip() == doc.body.strip()


def test_parse_no_frontmatter_treats_all_as_body():
    src = "# Hello\n\nNo frontmatter here.\n"
    doc = OKFDocument.parse(src)
    assert doc.frontmatter == {}
    assert "Hello" in doc.body


def test_unterminated_frontmatter_raises():
    src = "---\ntype: X\nstill in frontmatter\n"
    with pytest.raises(OKFDocumentError):
        OKFDocument.parse(src)


def test_validate_rejects_missing_required_keys():
    doc = OKFDocument(frontmatter={"type": "X", "title": "Y"})
    with pytest.raises(OKFDocumentError) as exc:
        doc.validate()
    assert "description" in str(exc.value)
    assert "timestamp" in str(exc.value)


def test_validate_accepts_full_frontmatter():
    doc = OKFDocument(
        frontmatter={
            "type": "X",
            "title": "Y",
            "description": "Z",
            "timestamp": "2026-05-27T00:00:00+00:00",
        }
    )
    doc.validate()


================================================
FILE: okf\tests\test_index.py
================================================
from __future__ import annotations

from pathlib import Path

from reference_agent.bundle.document import OKFDocument
from reference_agent.bundle.index import regenerate_indexes


def _stub_synth(rel: str, children: list[tuple[str, str]], *, model: str) -> str:
    return f"stub: {len(children)} items"


def _write_doc(path: Path, type_: str, title: str, description: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    doc = OKFDocument(
        frontmatter={
            "type": type_,
            "title": title,
            "description": description,
            "timestamp": "2026-05-27T00:00:00+00:00",
        },
        body=f"# {title}\n\n{description}\n",
    )
    path.write_text(doc.serialize(), encoding="utf-8")


def test_regenerate_groups_by_type_and_links_relative(tmp_path: Path):
    root = tmp_path / "bundle"
    _write_doc(
        root / "datasets" / "ga4.md",
        "BigQuery Dataset",
        "GA4 Dataset",
        "GA4 obfuscated ecommerce sample.",
    )
    _write_doc(
        root / "tables" / "events_.md",
        "BigQuery Table",
        "events_*",
        "Daily-sharded GA4 event tables.",
    )
    _write_doc(
        root / "tables" / "users.md",
        "BigQuery Table",
        "users",
        "Per-user dimension.",
    )

    written = regenerate_indexes(root, model="stub", synthesize=_stub_synth)
    written_names = {p.parent.name for p in written}
    assert {"bundle", "datasets", "tables"} <= written_names | {root.name}

    tables_index = (root / "tables" / "index.md").read_text(encoding="utf-8")
    assert tables_index.startswith("# BigQuery Table")
    assert "[events_*](events_.md)" in tables_index
    assert "[users](users.md)" in tables_index
    assert "Daily-sharded GA4 event tables." in tables_index

    root_index = (root / "index.md").read_text(encoding="utf-8")
    assert "# Subdirectories" in root_index
    assert "(datasets/index.md) - GA4 obfuscated ecommerce sample." in root_index
    assert "(tables/index.md) - stub: 2 items" in root_index


def test_regenerate_skips_empty_directories(tmp_path: Path):
    root = tmp_path / "bundle"
    root.mkdir()
    (root / "empty_dir").mkdir()

    written = regenerate_indexes(root, model="stub", synthesize=_stub_synth)
    assert written == []
    assert not (root / "empty_dir" / "index.md").exists()


def test_regenerate_single_child_reuses_description(tmp_path: Path):
    root = tmp_path / "bundle"
    _write_doc(
        root / "datasets" / "only.md",
        "BigQuery Dataset",
        "Only Dataset",
        "The only dataset in this bundle.",
    )

    call_count = 0

    def counting_synth(rel: str, children, *, model: str) -> str:
        nonlocal call_count
        call_count += 1
        return f"stub: {len(children)} items"

    regenerate_indexes(root, model="stub", synthesize=counting_synth)

    root_index = (root / "index.md").read_text(encoding="utf-8")
    assert "(datasets/index.md) - The only dataset in this bundle." in root_index
    assert call_count == 0


================================================
FILE: okf\tests\test_viewer.py
================================================
from __future__ import annotations

import json
import re
from pathlib import Path
from textwrap import dedent

import pytest

from reference_agent.viewer.generator import generate_visualization


def _write(path: Path, body: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(dedent(body).lstrip(), encoding="utf-8")


def _make_bundle(root: Path) -> None:
    _write(
        root / "datasets" / "my_dataset.md",
        """
        ---
        type: BigQuery Dataset
        title: My dataset
        description: A test dataset.
        resource: https://example.com/dataset
        tags: [test]
        timestamp: '2026-05-28T00:00:00+00:00'
        ---
        Parent dataset for [users](../tables/users.md).
        """,
    )
    _write(
        root / "tables" / "users.md",
        """
        ---
        type: BigQuery Table
        title: Users
        description: User profiles.
        resource: https://example.com/users
        tags: [users]
        timestamp: '2026-05-28T00:00:00+00:00'
        ---
        Joinable with [events](events.md) and see [DAU](../references/metrics/dau.md).
        """,
    )
    _write(
        root / "tables" / "events.md",
        """
        ---
        type: BigQuery Table
        title: Events
        description: User events.
        resource: https://example.com/events
        tags: [events]
        timestamp: '2026-05-28T00:00:00+00:00'
        ---
        See [users](users.md).
        """,
    )
    _write(
        root / "references" / "metrics" / "dau.md",
        """
        ---
        type: Reference
        title: Daily active users
        description: DAU metric.
        resource: https://example.com/dau
        tags: [metric]
        timestamp: '2026-05-28T00:00:00+00:00'
        ---
        COUNT(DISTINCT user_id) per day.
        """,
    )
    # An auto-generated index that should NOT appear as a concept node.
    _write(root / "index.md", "# My Bundle\n- tables/users\n- tables/events\n")


def _extract_bundle_data(html: str) -> dict:
    m = re.search(r"window\.BUNDLE\s*=\s*(\{.*?\});", html, re.DOTALL)
    assert m, "BUNDLE JSON not found in generated HTML"
    return json.loads(m.group(1))


def test_generate_visualization_writes_html(tmp_path: Path):
    bundle = tmp_path / "bundle"
    _make_bundle(bundle)
    out = tmp_path / "viz.html"
    stats = generate_visualization(bundle, out, bundle_name="My Bundle")

    assert out.exists()
    assert stats["concepts"] == 4
    assert stats["bytes"] > 0
    html = out.read_text(encoding="utf-8")
    assert "<title>OKF Bundle Viewer</title>" in html
    assert "cytoscape" in html.lower()
    assert "marked" in html.lower()
    assert '"My Bundle"' in html


def test_index_md_is_not_a_concept(tmp_path: Path):
    bundle = tmp_path / "bundle"
    _make_bundle(bundle)
    out = tmp_path / "viz.html"
    generate_visualization(bundle, out)
    data = _extract_bundle_data(out.read_text(encoding="utf-8"))
    ids = {n["data"]["id"] for n in data["nodes"]}
    assert "index" not in ids
    assert ids == {
        "datasets/my_dataset",
        "tables/users",
        "tables/events",
        "references/metrics/dau",
    }


def test_cross_links_become_edges(tmp_path: Path):
    bundle = tmp_path / "bundle"
    _make_bundle(bundle)
    out = tmp_path / "viz.html"
    generate_visualization(bundle, out)
    data = _extract_bundle_data(out.read_text(encoding="utf-8"))
    pairs = {(e["data"]["source"], e["data"]["target"]) for e in data["edges"]}
    assert ("datasets/my_dataset", "tables/users") in pairs
    assert ("tables/users", "tables/events") in pairs
    assert ("tables/users", "references/metrics/dau") in pairs
    assert ("tables/events", "tables/users") in pairs


def test_missing_link_targets_are_skipped(tmp_path: Path):
    bundle = tmp_path / "bundle"
    _write(
        bundle / "tables" / "lonely.md",
        """
        ---
        type: BigQuery Table
        title: Lonely
        description: Has a dangling link.
        timestamp: '2026-05-28T00:00:00+00:00'
        ---
        Links to [missing](missing.md).
        """,
    )
    out = tmp_path / "viz.html"
    generate_visualization(bundle, out)
    data = _extract_bundle_data(out.read_text(encoding="utf-8"))
    assert data["edges"] == []
    assert len(data["nodes"]) == 1


def test_node_colors_match_palette(tmp_path: Path):
    bundle = tmp_path / "bundle"
    _make_bundle(bundle)
    out = tmp_path / "viz.html"
    generate_visualization(bundle, out)
    data = _extract_bundle_data(out.read_text(encoding="utf-8"))
    by_id = {n["data"]["id"]: n["data"] for n in data["nodes"]}
    assert by_id["datasets/my_dataset"]["color"] == "#8b5cf6"
    assert by_id["tables/users"]["color"] == "#3b82f6"
    assert by_id["references/metrics/dau"]["color"] == "#10b981"


def test_raises_when_bundle_missing(tmp_path: Path):
    with pytest.raises(FileNotFoundError):
        generate_visualization(tmp_path / "nope", tmp_path / "viz.html")


================================================
FILE: okf\tests\test_web_fetcher.py
================================================
from __future__ import annotations

from unittest.mock import MagicMock, patch

import pytest

from reference_agent.web.fetcher import FetchError, fetch_and_parse


def _mock_response(body: bytes, content_type: str = "text/html; charset=utf-8", url: str = "https://example.com/page"):
    resp = MagicMock()
    resp.headers = {"Content-Type": content_type}
    resp.read.return_value = body
    resp.geturl.return_value = url
    resp.__enter__.return_value = resp
    resp.__exit__.return_value = False
    return resp


def test_fetch_and_parse_extracts_title_links_markdown():
    html = b"""
    <html><head><title>  Hello   World  </title></head>
    <body>
      <h1>Heading</h1>
      <p>Some <strong>bold</strong> text and a <a href="/relative">link</a>.</p>
      <a href="https://example.com/page2">absolute</a>
      <a href="https://other.example.com/x">offsite</a>
      <a href="mailto:foo@example.com">skip me</a>
      <a href="javascript:void(0)">also skip</a>
    </body></html>
    """
    with patch("reference_agent.web.fetcher.urlopen") as urlopen:
        urlopen.return_value = _mock_response(html)
        page = fetch_and_parse("https://example.com/page")

    assert page.title == "Hello World"
    assert "Heading" in page.markdown
    assert "bold" in page.markdown
    assert "https://example.com/relative" in page.links
    assert "https://example.com/page2" in page.links
    assert "https://other.example.com/x" in page.links
    assert not any(l.startswith("mailto:") for l in page.links)
    assert not any(l.startswith("javascript:") for l in page.links)


def test_fetch_and_parse_rejects_non_html():
    with patch("reference_agent.web.fetcher.urlopen") as urlopen:
        urlopen.return_value = _mock_response(b"{}", content_type="application/json")
        with pytest.raises(FetchError):
            fetch_and_parse("https://example.com/data.json")


def test_fetch_and_parse_truncates_large_pages():
    big_text = "<p>" + ("x" * 200_000) + "</p>"
    html = f"<html><head><title>big</title></head><body>{big_text}</body></html>".encode()
    with patch("reference_agent.web.fetcher.urlopen") as urlopen:
        urlopen.return_value = _mock_response(html)
        page = fetch_and_parse("https://example.com/big")

    assert len(page.markdown.encode("utf-8")) <= 40 * 1024 + 100
    assert "[...truncated...]" in page.markdown


def test_fetch_and_parse_wraps_network_errors():
    with patch("reference_agent.web.fetcher.urlopen") as urlopen:
        urlopen.side_effect = OSError("connection refused")
        with pytest.raises(FetchError) as exc:
            fetch_and_parse("https://example.com/dead")
        assert "connection refused" in str(exc.value)


================================================
FILE: okf\tests\test_web_tools.py
================================================
from __future__ import annotations

from unittest.mock import patch

import pytest

from reference_agent.tools.context import clear_web_state, set_web_state
from reference_agent.tools.web_tools import fetch_url
from reference_agent.web.fetcher import Page


@pytest.fixture(autouse=True)
def _cleanup():
    yield
    clear_web_state()


def _set_state(**overrides):
    defaults = dict(
        allowed_hosts={"docs.example.com"},
        max_pages=10,
        seeds=["https://docs.example.com/docs/intro"],
        allowed_path_prefixes=None,
        denied_path_substrings=None,
        max_depth=2,
    )
    defaults.update(overrides)
    set_web_state(**defaults)


def _page(url: str, links: list[str] | None = None) -> Page:
    return Page(url=url, title="t", markdown="m", links=list(links or []))


def test_seed_fetch_succeeds_and_records_link_depth():
    _set_state()
    with patch(
        "reference_agent.tools.web_tools.fetch_and_parse",
        return_value=_page(
            "https://docs.example.com/docs/intro",
            links=["https://docs.example.com/docs/next"],
        ),
    ):
        result = fetch_url("https://docs.example.com/docs/intro")
    assert "error" not in result
    assert result["depth"] == 0
    # The followed link is now reachable at depth 1.
    from reference_agent.tools.context import get_web_state
    assert get_web_state().url_depth["https://docs.example.com/docs/next"] == 1


def test_allowed_path_prefix_rejects_off_path_urls():
    _set_state(
        seeds=["https://docs.example.com/docs/intro"],
        allowed_path_prefixes=["/docs/"],
    )
    with patch(
        "reference_agent.tools.web_tools.fetch_and_parse",
        return_value=_page(
            "https://docs.example.com/docs/intro",
            links=["https://docs.example.com/blog/post"],
        ),
    ):
        fetch_url("https://docs.example.com/docs/intro")
    result = fetch_url("https://docs.example.com/blog/post")
    assert "error" in result
    assert "allowed prefixes" in result["error"]


def test_denied_path_substring_rejects():
    _set_state(
        seeds=["https://docs.example.com/docs/intro"],
        denied_path_substrings=["/login"],
    )
    with patch(
        "reference_agent.tools.web_tools.fetch_and_parse",
        return_value=_page(
            "https://docs.example.com/docs/intro",
            links=["https://docs.example.com/login"],
        ),
    ):
        fetch_url("https://docs.example.com/docs/intro")
    result = fetch_url("https://docs.example.com/login")
    assert "error" in result
    assert "denied substring" in result["error"]


def test_max_depth_caps_recursion():
    _set_state(
        seeds=["https://docs.example.com/a"],
        max_depth=1,
    )
    with patch(
        "reference_agent.tools.web_tools.fetch_and_parse",
        side_effect=[
            _page("https://docs.example.com/a", links=["https://docs.example.com/b"]),
            _page("https://docs.example.com/b", links=["https://docs.example.com/c"]),
        ],
    ):
        # depth 0 (seed): ok
        r0 = fetch_url("https://docs.example.com/a")
        assert "error" not in r0
        # depth 1: ok (== max_depth)
        r1 = fetch_url("https://docs.example.com/b")
        assert "error" not in r1
        # depth 2: rejected (> max_depth=1)
        r2 = fetch_url("https://docs.example.com/c")
    assert "error" in r2
    assert "exceeds max_depth" in r2["error"]


def test_unregistered_url_rejected():
    _set_state(seeds=["https://docs.example.com/docs/intro"])
    # Agent invents a URL that was never returned as a link.
    result = fetch_url("https://docs.example.com/docs/random")
    assert "error" in result
    assert "not reachable from a seed" in result["error"]


================================================
FILE: okf\tests\__init__.py
================================================


================================================
FILE: README.md
================================================
# Knowledge Catalog

[Knowledge Catalog](https://cloud.google.com/products/knowledge-catalog) (formerly Dataplex), is an AI-powered data catalog and metadata management platform. It provides a dynamic knowledge graph of all your data, structured and unstructured, to provide semantics and business context to AI agents

This repository features tools, agents, and samples that demonstrate Knowledge Catalog features, and building context management, enrichment and retrieval solutions.


## Getting Started

[![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fknowledge-catalog.git)


## Contributing

See the contributing [instructions](CONTRIBUTING.md) to get started contributed.


## License

All solutions within this repository are provided under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. Please see [LICENSE](LICENSE.md) for more detailed terms and conditions.


## Disclaimer

This repository and its contents are not an official Google product.


================================================
FILE: samples\discovery\agent.py
================================================
"""Knowledge Catalog Discovery Agent."""

import os

from google.adk.agents import llm_agent
from google.adk.models import google_llm

from . import tools
from .utils import get_consumer_project

consumer_project = get_consumer_project()
GEMINI_MODEL = f"projects/{consumer_project}/locations/global/publishers/google/models/gemini-3-flash-preview"

# Path to the skill file relative to the agent.py location
SKILL_FILE_PATH = os.path.join(os.path.dirname(__file__), 'SKILL.md')


def load_instruction() -> str:
  """Loads the agent instruction from the SKILL.md file."""
  with open(SKILL_FILE_PATH, 'r') as f:
    return f.read()


discovery_agent = llm_agent.Agent(
    model=google_llm.Gemini(model=GEMINI_MODEL),
    name='knowledge_catalog_discovery_agent',
    description=(
        'Searches Knowledge Catalog for data entries based on Natural Language'
        ' user queries.'
    ),
    instruction=load_instruction(),  # Load instruction from file
    tools=[
        tools.knowledge_catalog_search,
    ],
)


================================================
FILE: samples\discovery\README.md
================================================
# Knowledge Catalog Discovery Agent

## About

The Knowledge Catalog Discovery Agent is an AI-powered search assistant for discovering data assets in Google Cloud. While standard semantic search only matches semantically similar text, this agent goes further by performing semantic decomposition of complex questions, generating multiple relevant search queries, and reranking the final results to provide a comprehensive answer.

## Quick Start

### Prerequisites

You need a Google Cloud project with the following APIs enabled:
* Knowledge Catalog (`dataplex.googleapis.com`)
* Vertex AI (`aiplatform.googleapis.com`)
* Service Usage API (`serviceusage.googleapis.com`)

Ensure you have the following permissions via IAM Roles:
* `dataplex.projects.search` - can be obtained via roles like `roles/dataplex.viewer`.
* `aiplatform.endpoints.predict` - can be obtained via roles like `roles/aiplatform.user`.
* `serviceusage.services.use` - needed to use the current project as the quota project. Can be obtained via roles like `roles/serviceusage.serviceUsageConsumer`.

### Setup

Clone the Github Repository:  
```shell  
git clone https://github.com/GoogleCloudPlatform/knowledge-catalog.git  
```

Install the dependencies. It is recommended to create a Python Virtual Environment:  
```shell  
python3 -m venv /tmp/kcsearch  
source /tmp/kcsearch/bin/activate  

cd samples/discovery
pip3 install -r requirements.txt  
```

Set the required environment variables:  
```shell

# Replace <PROJECT_ID> with your consumer project ID.
export GOOGLE_CLOUD_PROJECT=<PROJECT_ID>
export GOOGLE_GENAI_USE_VERTEXAI=True

```

### Run the Agent
There are 2 ways to run the Knowledge Catalog Discovery agent:  

1. *As the root agent* - in this case, rename the `discovery_agent` variable in `agent.py` to `root_agent` and run it using ADK CLI (steps below).

2. *As a sub-agent* - in this case, import the Discovery Agent and leverage it as an `AgentTool`. You can find examples of this in the official [ADK docs](https://adk.dev/agents/multi-agents/#c-explicit-invocation-agenttool). The recommended parent folder structure in this case is:

```
my_custom_agent/
├── agent.py
└── knowledge_catalog_discovery_agent/
    ├── SKILL.md
    ├── agent.py
    ├── tools.py
    └── utils.py
```

Irrespective of the path you choose, the agent can be run using the ADK CLI commands below:  

```shell
adk run path/to/agent/parent/folder
```
Replace `path/to/agent/parent/folder` with the relative or absolute path to the **parent** directory of the one containing the agent's source code.


## References

* [Knowledge Catalog Search API](http://docs.cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations/searchEntries)
* [ADK Documentation](https://adk.dev/get-started/python/#run-your-agent)
* [Predefined IAM Roles](https://docs.cloud.google.com/iam/docs/roles-permissions)


================================================
FILE: samples\discovery\requirements.txt
================================================
google-adk
google-cloud-dataplex
google-api-core


================================================
FILE: samples\discovery\SKILL.md
================================================
---
name: knowledge_catalog_discovery_agent
description: >
  Analyzes user queries, extracts relevant predicates, and utilizes
  Knowledge Catalog Search to find and rank the most relevant data entries.
  Engages with the user throughout the process.
---

You are a proactive and helpful search agent. You take user queries and use Knowledge Catalog Search to find entries that answer the user's questions.

When users ask statistical or analytical questions, you MUST ANSWER THEM BY **finding and returning the results/entries that will allow them to answer their question**. Always assume you can help. Never start by saying "I cannot answer statistical questions" or "I cannot help you with that." Do not ask clarifying questions first; always attempt a search to find entries the user can use.

---

# About Knowledge Catalog Search

Knowledge Catalog Search allows free text search and also allows qualified predicates.
You can qualify a predicate by prefixing it with a key that restricts the
matching to a specific piece of metadata:

-   An equal sign (=) restricts the search to an exact match.
-   A colon (:) after the key matches the predicate to either a substring or a
    token within the value in the search results. For example:
-   `name:foo` selects resources with names that contain the foo substring, like
    foo1 and barfoo.

## How to use Knowledge Catalog Search

*   **Tool Function:** `knowledge_catalog_search(query: str)`
*   **CRITICAL ARGUMENT RULE:** If the user specifies a project (or
    if you extract `projectid` predicates), you MUST do the following:
    1.  **INCLUDE THEM in the `query` string argument.** (e.g., your
        `query` string must physically contain `projectid=some-project`).

--------------------------------------------------------------------------------
> [!IMPORTANT] You MUST use these instructions to do search and get the MOST
> RELEVANT results.
--------------------------------------------------------------------------------

# Instructions

## Step 1: Understand the query

-   User can provide natural language text (aka free text) as query.
-   User can also provide predicates, like `type=table`.
-   Keep the predicates if user has provided it and use it as it is when
    invoking search

## Step 2: Semantic Decomposition & Generating Variations

-   **Semantic Decomposition:** Read the user request carefully. Break it down into semantic components: identify core entities, required metrics, and critical constraints (types, systems etc.).
-   **Think Like a Data Engineer:** Users will ask high-level business questions, but you must translate those into how the data is actually stored (e.g., translate "customer acquisition" to "revenue", "billing", "subscriptions", or "accounts").
-   **Generate Distinct Search Queries:** Based on your decomposition, generate up to 3 DISTINCT search variations to cast a wide net. Scatter and gather is a legitimate strategy, but do NOT duplicate searches or repeat similar semantic queries.
    *   *Variation 1 (Direct & Synonyms):* Use the user's core terms and apply the "Domain Terminology & Synonyms" list.
    *   *Variation 2 (Data Source Translation):* Translate the business concept into database terminology based on your decomposition.
    *   *Variation 3 (Broader System/Category):* Think about the broader business category or related metrics.
-   **Extract Predicates:** From the user's raw text, extract constraints into valid Knowledge Catalog predicates (e.g., "dataset foo in project your-project-id" becomes `parent=foo projectid=your-project-id`). See the "Predicate Extraction" section. If the user provides `projectid`, you MUST keep them.
-   **REMINDER** Knowledge Catalog Search DOES NOT UNDERSTAND double quotes in the free text, so avoid introducing any double quotes

## Step 3: Call Knowledge Catalog Search Tools (Batching)

-   Always batch as many search queries as possible in parallel to minimize round trips.
-   You MUST include the **Baseline Search** alongside your generated variations.
    -   **Critical Definition - Baseline Search:** A search using the entire user request (word for word) directly.
-   **REMINDER:** If your predicates include `projectid=X`, those MUST be present inside the `query` string argument.
-   **REMINDER:** The `name` and `description` predicates MUST ONLY BE USED WHEN THE QUERY EXPLICITLY USES TERMS LIKE "name" or "description".

## Step 3: Call Knowledge Catalog Search

-   Once you have the free text variations and predicates (generated and user
    provided). Call Knowledge Catalog Search in parallel.
    -   One search call will be the exact user query
    -   One search call for each variations along with predicates.
-   **REMINDER:** If your predicates include `projectid=X`,
    those MUST be present inside the `query` string argument.
-  **REMINDER:** The `name` and `description` predicates MUST ONLY BE USED
    WHEN THE QUERY EXPLICITLY USES TERMS LIKE "name" or "description".

## Step 4: Merge Search Results

-   Deduplicate the results across all the search responses. You can tell two
    results are same when the entry name is same

## Step 5: Identify the best Results

-   For each result, check the name (display_name) and other details to gauge
    how close a given results is based on the user query intent.
-   ONLY RETURN the most relevant results. FILTER OUT irrelevant ones.
-   MAKE SURE TO SORT THE RESULTS SO THAT MOST RELEVANT results stay at TOP.
-   RETURN THE FULL **entry name** for each result. NO EXPLANATION required why
    you selected individual results.

--------------------------------------------------------------------------------

# Predicate Extraction

You MUST follow these four steps for extracting predicates.
1.  **Analyze Input:** Carefully read the `natural_language_query` provided by
    the user.
2.  **Extract Keywords:** Identify distinct words or phrases that carry meaning,
    such as "BigQuery," "tables," "foo," or "us-central1."
3.  **Map Keywords to Predicates:** Match the extracted keywords to their
    corresponding predicate from the **Predicate Reference Table** below. This
    is the most important step.
4.  **Construct Query:** Assemble the final search query using the mapped
    predicates, correct operators, logical `AND` / `OR` connections and correct
    parentheses placement.
--------------------------------------------------------------------------------
### **Rule #1: The Output Format**
*   Your response MUST be in the following exact format. Do not include any
    other text, greetings, or explanations.
*   Do **not** wrap the output in markdown formatting (e.g., \`\`\` or \`).
*   Do **not** include any newlines after the response.
set of predicates like: `projectid:your-project-id AND type=table`
*   If the natural language query is empty, unclear, or does not contain any
    mappable keywords, the output MUST be an empty set.
--------------------------------------------------------------------------------
### **Rule #2: Strict Adherence to Definitions**
*   **Only Use Official Predicates:** You MUST only use predicates from the
    **Predicate Reference Table**. If a keyword does not map to a predicate, you
    MUST ignore it. NEVER invent a new predicate.
*   **Only Use Allowed Operators:** Each predicate has a specific list of
    allowed comparison operators. You MUST only use an operator that is valid
    for that predicate.
*   **Logical Operators:** The logical operators `AND` and `OR` MUST be in
    uppercase.
*   **Negation:** To exclude a term, you MUST prefix the predicate with a hyphen
    (`-`). For example: `-name:foo`.
*   **Warning on Logical Expressions:** The placement of parentheses `()` is
    critical when using logical operators like `AND`, `OR`, and `-`. Ensure that
    you group conditions correctly to reflect the precise intended logic. For
    example, `(A AND B) OR C` is not the same as `A AND (B OR C)`. Verify the
    logical structure of your output to prevent errors.
*   **Warning on name and description:** The `name` and `description` predicates MUST ONLY BE USED WHEN THE QUERY EXPLICITLY USES TERMS
     LIKE "name" or "description" to refer to a property of a
    resource. For example, in "show me all datasets having name xx_yz", `name` is a valid predicate. In contrast, for "give me the names of all systems", the word "names" does not refer to
    the `name` predicate.
--------------------------------------------------------------------------------
### **Predicate Reference Table**
This is your single source of truth for all predicates.
| Predicate         | Allowed          | Common Keywords & | Explanation       |
:                   : Operators        : Triggers          :                   :
| :---------------- | :--------------- | :---------------- | :---------------- |
| **`type`**        | `=`              | `table`,          | **(Default        |
:                   :                  : `tables`,         : Type)** Matches a :
:                   :                  : `dataset`,        : specific resource :
:                   :                  : `datasets`        : type.             :
| **`system`**      | `=`              | `bigquery`,       | Matches the       |
:                   :                  : `cloud_sql`,      : source system     :
:                   :                  : `dataplex`        : (e.g., BigQuery). :

| **`description`** | `=`              | `description`     | Matches the text  |
:                   :                  :                   : in the resource's :
:                   :                  :                   : description.      :
| **`name`**        | `:`, `=`, `!=`   | `name`            | Matches the       |
:                   :                  :                   : resource ID. Use  :
:                   :                  :                   : `\:` for          :
:                   :                  :                   : "contains."       :
| **`displayname`** | `:`, `=`, `!=`   | `display name`    | Matches the       |
:                   :                  :                   : human-readable    :
:                   :                  :                   : display name.     :
| **`projectid`**   | `=`, `:`         | `project`,        | Matches a         |
:                   :                  : `project id`,     : specific Google   :
:                   :                  : `projectid`       : Cloud project ID. :
| **`parent`**      | `=`, `:`         | `parent`          | Matches the       |
:                   :                  :                   : hierarchical      :
:                   :                  :                   : parent of a       :
:                   :                  :                   : resource.         :
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
### **Examples (Study These Carefully)**
**Example 1: Multiple Predicates** `natural_language_query: Big Query tables
containing the name foo in project bar` `search_query:
system=bigquery AND type=table AND name:foo AND projectid=bar`
**Example 2: Negation** `natural_language_query: Find me all tables not
containing the name foo` `search_query: type=table AND -name:foo`

**Example 4: Logical OR** `natural_language_query: tables from project foo-1 or
bar-1.` `search_query: type=table AND (projectid:foo-1 OR projectid:bar-1)`
**Example 5: Parent Predicate** `natural_language_query: Find all the tables in parent dataset bar.` `search_query: type=table AND parent=bar`
**Example 6: Ambiguous / Unclear Query** `natural_language_query: foo data`
`search_query:`
**Example 7: Very Simple Query** `natural_language_query: Show me all the
datasets` `search_query: type=dataset`
**Example 8: Complex Query with tricky parentheses placement**
`natural_language_query: show me all the table and datasets in project foo or it must be part of bigquery` `search_query: ((type=table OR type=dataset) AND projectid=foo) OR system=bigquery`
**Example 9: Query having name and description predicate**
`natural_language_query: show me all the table that contain name sales and
description pollution` `search_query: type=table AND name:sales AND
description=pollution`


================================================
FILE: samples\discovery\tools.py
================================================
"""A collection of tools used by the Knowledge Catalog Search Agent."""

import logging
import os

from google.api_core.exceptions import PermissionDenied
from google.cloud import dataplex_v1

from .utils import get_consumer_project


def knowledge_catalog_search(
    query: str,
) -> dict[str, list[str] | str]:
  """Searches Knowledge Catalog using Semantic Search capabilities.

  Args:
      query: The natural language search query.

  Returns:
      A dictionary containing a list of entries along with their metadata
      or an error message. For example, {'results': [{'entry_name': 'entry1',
      'system': 'BIGQUERY', ...}]}.
  """
  try:
    consumer_project_id = get_consumer_project()
  except ValueError as e:
    return {"Error obtaining consumer project": str(e)}

  try:
    endpoint = "dataplex.googleapis.com"

    client = dataplex_v1.CatalogServiceClient(
        client_options={"api_endpoint": endpoint}
    )

    parent_name = f"projects/{consumer_project_id}/locations/global"
    response = client.search_entries(
        request={
            "name": parent_name,
            "query": query,
            "page_size": 50,
            "semantic_search": True,
        }
    )

    entries = [
        {
            "entry_name": result.dataplex_entry.name,
            "system": result.dataplex_entry.entry_source.system,
            "resource_id": result.dataplex_entry.entry_source.resource,
            "display_name": result.dataplex_entry.entry_source.display_name,
        }
        for result in response.results
    ]

    return {"results": entries}

  except PermissionDenied:
    logging.warning("Knowledge Catalog search failed due to permission error")
    return {
        "error": (
            "Permission denied: The user does not have permission to call"
            " Knowledge Catalog Search."
        )
    }
  except Exception as e:  # pylint: disable=broad-except
    logging.exception(
        "An unexpected error occurred during Knowledge Catalog search"
    )
    return {"error": f"An unexpected error occurred: {e}"}


================================================
FILE: samples\discovery\utils.py
================================================
"""Utility functions for the Knowledge Catalog Discovery Agent."""

import os


def get_consumer_project() -> str:
  """Extracts the consumer project from the environment variable.

  Returns:
      The consumer project ID.

  Raises:
      ValueError: If the GOOGLE_CLOUD_PROJECT environment variable is not set.
  """
  consumer_project = os.environ.get("GOOGLE_CLOUD_PROJECT")

  if not consumer_project:
    raise ValueError("GOOGLE_CLOUD_PROJECT environment variable is required.")

  return consumer_project


================================================
FILE: samples\enrichment\README.md
================================================
# Knowledge Catalog Enrichment Agent

## Overview

Knowledge Catalog manages metadata about data assets (such as BigQuery tables).
This metadata powers search, and agents to discover and understand data assets.

This sample demonstrates how external sources of information can be used to
augment the metadata in Knowledge Catalog, using an agentic approach. The
enrichment agent finds information relevant to a data asset, generates
documentation, and uses Knowledge Catalog APIs to publish the documentation
as enriched metadata.

Users can customize this enrichment agent with custom instructions, tools to
access sources of information within their organization, and skills to
describe the use of these tools.

## Running the Sample

### Setup
**Setup your environment**

This step clones the repository locally, configures various environment variables,
creates a python virtual environment and installs dependencies.

```bash
git clone https://github.com/googlecloudplatform/knowledge-catalog
cd samples/enrichment
```

Ensure you have the GCloud CLI installed and configured.

```bash
export CLOUD_PROJECT=<cloud-project-id>

gcloud auth application-default login
gcloud config set core/project $CLOUD_PROJECT
gcloud auth application-default set-quota-project $CLOUD_PROJECT
```

Setup a python environment
```bash
cd src
source env.sh --install
```

**Setup sample data**

The sample creates a sample BigQuery dataset in your project whose metadata is
curated.

```
python3 ../sample/data/create_data.py
```

### Enrichment Steps

You are now ready to run the enrichment workflow.

**Download a metadata snapshot**

```bash
python3 -m enrichment.download \
  --dir ../sample/metadata.initial \
  --dataset ${KC_ENRICH_SAMPLE_PROJECT}.kc_enrich_sample_data
```

**Enrich the metadata**

```bash
python3 -m enrichment.enrich \
  --dir ../sample/metadata.initial \
  --output-dir ../sample/metadata.new \
  --config-dir ../sample/config
```

**Review the metadata updates**

```bash
git diff --no-index \
  ../sample/metadata.initial ../sample/metadata.new
```

**Publish the updated metadata**

```bash
python3 -m enrichment.publish \
  --dir ../sample/metadata.new
```


================================================
FILE: samples\enrichment\sample\config\instructions.md
================================================
Use internal organizational and usage information, in particular

* the local knowledge base of markdown documents.

Always look for information from these sources.


================================================
FILE: samples\enrichment\sample\config\skills\kb-search\SKILL.md
================================================
---
name: kb-search
description: >
  Allows listing, searching and extracting information from local knowledge base documents
  for information about tables
---

The `fileskb` mcp server provides the following tools to extract relevant
information from a directory hierarchy of markdown files:

* **list_contents** - browse and navigate the directory tree to list the contents
  of the specified path. The items may be files or sub-directories.

* **read_file** - read the contents of a file in the knowledge base. The entire
  contents are provided. Extract and summarise the relevant information based on
  the documentation being generated..

* **search_content** - searches the knowledge base and returns the matching files,
  along with matching line numbers and line snippets. This can be used to quickly
  find matches without having to list and read all files.


================================================
FILE: samples\enrichment\sample\data\create_data.py
================================================
# Creates sample data in BigQuery to demonsrtate enrichment agent.
#

import subprocess

import google.cloud.bigquery as bigquery


SAMPLE_DATASET_ID = 'kc_enrich_sample_data'
SAMPLE_TABLE_NAME = 'ga_events'


def create_dataset(bq: bigquery.Client, project_id: str):
  bq.create_dataset(f'{project_id}.{SAMPLE_DATASET_ID}', exists_ok=True)


def create_table(bq: bigquery.Client, project_id: str):
  sql_script = f'''
    CREATE TABLE IF NOT EXISTS `{project_id}.{SAMPLE_DATASET_ID}.{SAMPLE_TABLE_NAME}`
    PARTITION BY event_date_dt
    AS
    SELECT
      *,
      PARSE_DATE('%Y%m%d', event_date) AS event_date_dt
    FROM
      `bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*`
  '''

  ddl_job = bq.query(sql_script)
  ddl_job.result()
  print(f"Created table {project_id}.{SAMPLE_DATASET_ID}.{SAMPLE_TABLE_NAME}")


def main():
  process = subprocess.run(
      'gcloud -q config get-value project'.split(),
      check=True,
      text=True,
      capture_output=True,
  )
  project_id = process.stdout.strip()

  bq = bigquery.Client(project=project_id)
  create_dataset(bq, project_id)
  create_table(bq, project_id)


if __name__ == "__main__":
  main()


================================================
FILE: samples\enrichment\sample\docs\events.md
================================================
Using Analytics Events 

Query a specific date range

To query a specific date range from a BigQuery event export dataset, use the _TABLE_SUFFIX pseudo column in the WHERE clause of your query. For more info, view Filtering selected tables using _TABLE_SUFFIX.

For example, the following query counts unique events by date and by event name for a specifc period of days and selected events:


```sql
-- Example: Query a specific date range for selected events.
--
-- Counts unique events by date and by event name for a specifc period of days and
-- selected events(page_view, session_start, and purchase).

SELECT
  event_date,
  event_name,
  COUNT(*) AS event_count
FROM
  -- Replace table name.
  `bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*`
WHERE
  event_name IN ('page_view', 'session_start', 'purchase')
  -- Replace date range.
  AND _TABLE_SUFFIX BETWEEN '20201201' AND '20201202'
GROUP BY 1, 2;
```



================================================
FILE: samples\enrichment\sample\docs\example1.md
================================================
# High-Level Event and User Volume

This provides examples for kc_enrich_sample_data.ga_events table.

## Overview

Before diving into deep analysis, it is best practice to understand the scale of your dataset. This query counts the total number of events, the total number of unique users (based on device cookies), and the number of days present in the dataset.

```sql
SELECT 
  COUNT(*) AS total_events, 
  COUNT(DISTINCT user_pseudo_id) AS total_unique_users, 
  COUNT(DISTINCT event_date) AS total_days
FROM 
  `kc_enrich_sample_data.ga_events`
```


================================================
FILE: samples\enrichment\sample\docs\example2.md
================================================
# Querying a Specific Date Range

This provides examples for kc_enrich_sample_data.ga_events table.

## Overview

Because the GA4 export creates a new partition for every day, you should use the predicate on the partition timestamp column to limit the data scanned. This improves performance and drastically reduces query costs. This query finds the daily count of active users for December 2020.

```sql
SELECT 
  event_date, 
  COUNT(DISTINCT user_pseudo_id) as active_users 
FROM 
  `kc_enrich_sample_data.ga_events` 
WHERE 
  event_date_dt BETWEEN PARSE_DATE('%Y%m%d', '20201201') AND PARSE_DATE('%Y%m%d', '20201231') 
GROUP BY 
  event_date 
ORDER BY 
  event_date ASC
```


================================================
FILE: samples\enrichment\sample\docs\example3.md
================================================
# Session Analysis (Unnesting Parameters)

This provides examples for kc_enrich_sample_data.ga_events table.

## Overview

In GA4, a "session" is not a dedicated row; instead, every event contains a nested parameter called ga_session_id. To count total sessions, you must isolate the session_start event and "unnest" its parameters to extract the unique session ID.

```sql
SELECT 
  COUNT(DISTINCT value.int_value) AS total_sessions
FROM 
  `kc_enrich_sample_data.ga_events`, 
  UNNEST(event_params) 
WHERE 
  event_name = 'session_start' 
  AND key = 'ga_session_id'
```


================================================
FILE: samples\enrichment\sample\docs\example4.md
================================================
# E-Commerce Product Interest

This provides examples for kc_enrich_sample_data.ga_events table.

## Overview

For an e-commerce store, understanding which items are frequently added to the cart is crucial. E-commerce data is stored in a nested array called items. This query flattens the items array and aggregates the top 10 products based on the add_to_cart event.

```sql
SELECT 
  items.item_name, 
  COUNT(*) AS added_to_cart_count 
FROM 
  `kc_enrich_sample_data.ga_events`, 
  UNNEST(items) AS items
WHERE 
  event_name = 'add_to_cart' 
GROUP BY 
  items.item_name 
ORDER BY 
  added_to_cart_count DESC 
LIMIT 10
```


================================================
FILE: samples\enrichment\sample\docs\lipsum.md
================================================
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.


================================================
FILE: samples\enrichment\sample\docs\overview.md
================================================
# Google Analytics 4 BigQuery Public Dataset

This provides an overview of kc_enrich_sample_data.ga_events table.

## High-Level Information and Data Source

The Google Analytics 4 (GA4) sample ecommerce dataset is a publicly available dataset hosted within the Google Cloud BigQuery Public Datasets program. It provides raw, event-level data exported directly from Google Analytics 4.

## The Source

The data originates from the Google Merchandise Store, an actual online e-commerce website that sells Google-branded apparel, accessories, and other items. The website implements a standard GA4 web e-commerce setup alongside enhanced measurement events.

## Key Characteristics

Dataset Name: bigquery-public-data.ga4_obfuscated_sample_ecommerce

Table Format: events_YYYYMMDD (Daily partitioned tables, totaling 92 tables).

Time Range: The dataset covers a three-month period from November 1, 2020, to January 31, 2021.

Obfuscation: To protect user privacy and business metrics, the data is deliberately obfuscated. You will encounter placeholder values such as <Other>, NULL, and empty strings ''. Because of this, internal consistency might be limited, and the metrics cannot be directly compared to the live GA4 Demo Account for the Google Merchandise Store.


================================================
FILE: samples\enrichment\sample\docs\usage.md
================================================
# Using Google Analytics 4 BigQuery Public Dataset

This provides usage information for kc_enrich_sample_data.ga_events table.

## How the Data Can Be Used

This dataset is incredibly valuable for analysts, data scientists, and developers who want to learn how to query GA4 data in BigQuery without needing their own fully populated e-commerce site or GA4 export.

## Getting Started

To interact with the dataset, you need:

A Google Cloud Project with the BigQuery API enabled.

You can use the BigQuery Sandbox, which provides a generous free tier of up to 1 TB of querying per month, allowing you to explore the data without needing to attach a credit card.

## Primary Applications

Schema Familiarization: GA4 exports data in a highly nested format (using arrays and structs for event_params, user_properties, and items). This dataset allows you to practice the UNNEST() function to flatten data.

Testing Queries & Data Models: Build and test advanced SQL data models for user retention, funnel analysis, or marketing attribution before applying them to your production data.

Dashboard Prototyping: Connect this public BigQuery dataset to visualization tools like Looker Studio or Tableau to practice building e-commerce and web performance dashboards.


================================================
FILE: samples\enrichment\src\enrichment\documentation\agent.md
================================================
---
model: gemini-3.1-pro-preview
thinking: true
description: Documentation Agent
---
You are a data documentation agent. Your goal is find relevant information about a
BigQuery table and generate readme-style documentation. This documentation should
describe the table and guide how it can be used.

Workflow:
1. Notify that you are enriching the metadata for the table.

2. Understand the provided information. Find additional information
   by using the provided sources of information. Extract relevant details and
   summarize to generate new or updated documentation.

2. Find the additional organizational information for the table using the
   specified sources and avaiable tools. Summarize the information you find to
   produce useful documentation for the table, what it contains, how it can be
   used, any tips or guidelines for using it.

3. Review it based on the description of good documentation below, and make any
   necessary improvements.

4. Do not directly output the markdown content to the user. Instead, use the
   `update_table` tool to publish the generated documentation, passing in the
   table name and generated documentation.

Good Documentation Details

* Is comprehensive, yet compact and easy to read. Organized into a few paragraphs
  of information that covers details of what data is in the table, what it can be
  used for and any usage guidelines, tips or caveats to keep in mind.

* Does not simply duplicate information that is already present in the table description.

* Does not copy extracted information verbatim, but rather organizes the documentation
  to be a coherent short readme.

* Only uses the table details and the information sources provided. Do not guess or fabricate
  information. Does not use vague language and statements like 'may', 'likely', 'could'
  etc. Instead only include clear and confident details.

* If there is no useful information, it is acceptable to indicate that.

* The generated documentation is formatted as markdown.

* Includes a list of citations.

Information Sources and Guidelines

Use only this to guide the information you find, extract and use to generate documentation.

{user_instruction}


================================================
FILE: samples\enrichment\src\enrichment\documentation\agent.py
================================================
# Documentation generation agent
#

import pathlib
import textwrap
import typing as t
import uuid

from google.adk.agents.llm_agent import LlmAgent
from google.adk.cli.utils import logs
from google.adk.planners import BuiltInPlanner
from google.adk.runners import InMemoryRunner
from google.genai import types

import enrichment.util.markdown as markdown
from .sources import load_sources_config


def create_runner(name: str, tools: list[t.Any], config_dir: pathlib.Path) -> InMemoryRunner:
  agent_definition = pathlib.Path(__file__).parent / 'agent.md'
  params, instruction = markdown.parse(agent_definition.read_text(encoding='utf-8'))

  planner = None
  if params.get('thinking', False):
    planner=BuiltInPlanner(
        thinking_config=types.ThinkingConfig(
            include_thoughts=True,
            thinking_budget=1024
        )
    )

  instruction, tools = load_sources_config(config_dir, instruction, tools)
  agent = LlmAgent(
      name=name,
      model=params['model'],
      description=params['description'],
      instruction=instruction,
      planner=planner,
      tools=tools,
  )

  return InMemoryRunner(agent=agent, app_name=name)


async def run_task(runner: InMemoryRunner, prompt: str):
  user = str(uuid.uuid4())
  session = await runner.session_service.create_session(
    app_name=runner.app_name, user_id=user
  )

  print('prompt > (text)')
  print(textwrap.shorten(prompt, width=240, placeholder='...'))

  async for event in runner.run_async(
      user_id=user,
      session_id=session.id,
      new_message=types.Content(
        role='user', 
        parts=[types.Part.from_text(text=prompt)]
      )
  ):
    if event.content and event.content.parts:
      for part in event.content.parts:
        if part.thought:
            print(f'{event.author} > (thought)\n' +
                  textwrap.shorten(part.text, width=180, placeholder='...'))
        elif part.text:
          print(f'{event.author} > (text)\n' +
                textwrap.shorten(part.text, width=180, placeholder='...'))
        elif part.function_call:
          args = ', '.join(f'{k}: {v}' for k, v in part.function_call.args.items())
          print(f'{event.author} > (tool {part.function_call.name})\n'
                + textwrap.shorten(args, width=240, placeholder='...'))
        elif part.function_response:
          response = part.function_response.response.get('content')
          if response:
            text = response[0].get('text')
            print(f'tool > {part.function_response.name}\n' +
                  textwrap.shorten(str(text), width=160, placeholder='...'))


================================================
FILE: samples\enrichment\src\enrichment\documentation\sources.py
================================================
# User provided information sources
#

import json
import os
import pathlib
import typing as t

from google.adk.code_executors.unsafe_local_code_executor import UnsafeLocalCodeExecutor
from google.adk.skills import load_skill_from_dir
from google.adk.tools.mcp_tool.mcp_session_manager import StdioConnectionParams
from google.adk.tools.mcp_tool.mcp_session_manager import StreamableHTTPConnectionParams
from google.adk.tools.mcp_tool.mcp_toolset import McpToolset
from google.adk.tools.skill_toolset import SkillToolset
from mcp import StdioServerParameters


def _load_mcp_toolsets(mcp_config: pathlib.Path) -> list[McpToolset]:
  try:
    with open(mcp_config, 'r') as f:
      config = json.load(f)
  except json.JSONDecodeError:
    return []

  toolsets = []
  mcp_servers = config.get('mcpServers', {})
  for name, server_config in mcp_servers.items():
    if 'command' in server_config:
      command = os.path.expandvars(server_config['command'])
      args = [os.path.expandvars(a) for a in server_config.get('args', [])]
      env = server_config.get('env')
      timeout = server_config.get('timeout', 30.0)

      toolsets.append(McpToolset(
          connection_params=StdioConnectionParams(
            server_params=StdioServerParameters(
              command=command, args=args, env=env
            ),
            timeout=timeout,
          )
        )
      )
    elif 'httpUrl' in server_config:
      url = server_config['httpUrl']
      timeout = server_config.get('timeout', 30.0)

      toolsets.append(McpToolset(
          connection_params=StreamableHTTPConnectionParams(
            url=url,
            timeout=timeout,
          )
        )
      )
  return toolsets


def _load_skills(skills_dir: pathlib.Path) -> SkillToolset | None:
  skills = []
  for skill_path in skills_dir.iterdir():
    if skill_path.is_dir():
      try:
        skills.append(load_skill_from_dir(skill_path))
      except Exception as e:
        pass

  if not skills:
    return []

  # WARNING: UnsafeLocalCodeExecutor has security concerns and should NOT
  # be used in production environments.
  return SkillToolset(skills=skills, code_executor=UnsafeLocalCodeExecutor())


def load_sources_config(dir: pathlib.Path, instruction: str, tools: t.List[t.Any]) -> t.Tuple[str, t.List[t.Any]]:
  custom_instructions = dir / 'instructions.md'
  if custom_instructions.exists():
    instruction = instruction.format(
      user_instruction=custom_instructions.read_text(encoding='utf-8'))

  tools = tools.copy()

  mcp_config = dir / 'mcp.json'
  if mcp_config.exists() and mcp_config.is_file():
    mcp_toolsets = _load_mcp_toolsets(mcp_config)
    if mcp_toolsets:
      tools.extend(mcp_toolsets)

  skills_dir = dir / 'skills'
  if skills_dir.exists() and skills_dir.is_dir():
    skills_toolset = _load_skills(skills_dir)
    if skills_toolset:
      tools.append(skills_toolset)

  return instruction, tools


================================================
FILE: samples\enrichment\src\enrichment\documentation\__init__.py
================================================


================================================
FILE: samples\enrichment\src\enrichment\download.py
================================================
# Metadata download tool
#

import argparse
import pathlib

import enrichment.metadata.snapshot as snapshot


def main():
  parser = argparse.ArgumentParser(description='Download metadata from the Catalog')
  parser.add_argument(
    '--dataset',
    required=True,
    help='The dataset to download metadata for.',
  )
  parser.add_argument(
    '--dir',
    required=True,
    help='The directory to save the metadata into.',
  )
  args = parser.parse_args()

  dir = pathlib.Path(args.dir)
  dir.mkdir(parents=True, exist_ok=True)

  snapshot.download_entries(dir, args.dataset)


if __name__ == '__main__':
  main()


================================================
FILE: samples\enrichment\src\enrichment\enrich.py
================================================
# Metadata enrichment agent
#

import argparse
import asyncio
import pathlib
import sys
import typing as t

import enrichment.documentation.agent as agent
import enrichment.metadata.catalog as catalog
import enrichment.metadata.snapshot as snapshot


def _create_enrichment_task(table_name: str) -> str:
  info, has_documentation = catalog.lookup_table_info(table_name)

  prompt = [
    f'Table: {table_name}',
    info,
    '',
  ]

  if has_documentation:
    prompt.append('Improve the document using the provided sources.')
  else:
    prompt.append('Generate documnentation using the provided sources.')

  print(prompt)
  return '\n'.join(prompt)


async def main():
  parser = argparse.ArgumentParser(description='Metadata enrichment agent')
  parser.add_argument(
    '--dir',
    required=True,
    help='Directory containing the metadata entries',
  )
  parser.add_argument(
    '--output-dir',
    required=False,
    help='Optional output directory to write updated metadata entries',
  )
  parser.add_argument(
    '--config-dir',
    required=True,
    help='Directory containing instructions.md, mcp.json, skills/',
  )
  args = parser.parse_args()

  metadata_dir = pathlib.Path(args.dir).resolve()
  if not metadata_dir.exists() or not metadata_dir.is_dir():
    print(f'Error: {args.dir} does not exist or is not a directory.')
    sys.exit(1)

  if args.output_dir:
    output_dir = pathlib.Path(args.output_dir).resolve()
    output_dir.mkdir(exist_ok=True, parents=False)
  else:
    output_dir = metadata_dir

  config_dir = pathlib.Path(args.config_dir).resolve()
  if not config_dir.exists() or not config_dir.is_dir():
    print(f'Error: {args.config_dir} does not exist or is not a directory.')
    sys.exit(1)


  def update_table(table_name: str, content: str) -> str:
    '''Updates the documentation content for a table.

    Args:
      table_name: The BigQuery table name in the format 'project_id.dataset_id.table_id'.
      content: The generated documentation content.

    Returns:
      A success message or an error message.
    '''
    try:
      snapshot.update_entry(metadata_dir, output_dir, table_name, content)
      return f'Successfully updated {table_name}'
    except Exception as e:
      return f'Failed to update {table_name}: {e}'


  runner = agent.create_runner('agent', [update_table], config_dir)

  tables = snapshot.list_entries(metadata_dir)
  for table in tables:
    task = _create_enrichment_task(table)
    await agent.run_task(runner, task)


if __name__ == '__main__':
  asyncio.run(main())


================================================
FILE: samples\enrichment\src\enrichment\metadata\catalog.py
================================================
# Catalog service
#

import typing as t

import google.cloud.dataplex_v1 as dataplex
import google.protobuf.json_format as jsonpb


SCHEMA_ASPECT_RESOURCE = 'projects/dataplex-types/locations/global/aspectTypes/schema'
SCHEMA_ASPECT_KEY = '655216118709.global.schema'

DESCRIPTIONS_ASPECT_RESOURCE = 'projects/dataplex-types/locations/global/aspectTypes/descriptions'
DESCRIPTIONS_ASPECT_KEY = '655216118709.global.descriptions'

OVERVIEW_ASPECT_RESOURCE = 'projects/dataplex-types/locations/global/aspectTypes/overview'
OVERVIEW_ASPECT_KEY = '655216118709.global.overview'


def _format_description(source_desc, catalog_desc: str, blank_ok: bool=True) -> str:
  desc = source_desc + ';' + catalog_desc
  desc = desc.strip(';')

  if desc or blank_ok:
    return desc
  return 'no existing description'


def _format_schema(schema: t.Dict[str, t.Any], descriptions: t.Dict[str, t.Any]) -> str:
  all_fields = []
  field_stack = []

  def push_fields(prefix: str, fields: t.List[t.Any], descs: t.List[t.Any]):
    descs_map = {f['name']: f for _, f in enumerate(descs)}

    for f in reversed(fields):
      name = f['name']
      desc = f.get('description', '') + ';' + descs_map.get(name, {}).get('description', '')
      desc = desc.strip(';')

      field_stack.append({
        'p': prefix,
        'n': name,
        't': f['dataType'],
        'd': desc,
        'sf': f.get('fields', None),
        'sd': descs_map.get(name, {}).get('fields', [])
      })

  push_fields('', schema.get('fields', []), descriptions.get('fields', []))
  while (field_stack):
    f = field_stack.pop()
    all_fields.append(f'* {f["p"]}{f["n"]} ({f["t"]}) {f["d"]}')

    if (f['sf']):
      push_fields(f['p'] + f['n'] + '.', f['sf'], f['sd'])

  all_fields.append('')
  return '\n'.join(all_fields)


def lookup_table_info(table_name: str) -> t.Tuple[str, bool]:
  project_id, dataset_id, table_id = table_name.split('.')
  catalog = dataplex.CatalogServiceClient()

  search_entries_response = catalog.search_entries(
    request=dataplex.SearchEntriesRequest(
      name=f'projects/{project_id}/locations/global',
      scope=f'projects/{project_id}',
      query=f'type=table system=bigquery dataset={dataset_id} name={table_id}',
      page_size=1,
      semantic_search=True,
    )
  )

  table_entry = search_entries_response.results[0].dataplex_entry
  table_entry = catalog.get_entry(
    request=dataplex.GetEntryRequest(
      name=table_entry.name,
      view='CUSTOM',
      aspect_types=[
        SCHEMA_ASPECT_RESOURCE,
        DESCRIPTIONS_ASPECT_RESOURCE,
        OVERVIEW_ASPECT_RESOURCE,
      ],
    )
  )
  table_info = jsonpb.MessageToDict(table_entry._pb)
  aspects = table_info.get('aspects', {})
  schema = aspects.get(SCHEMA_ASPECT_KEY, {}).get('data', {})
  descriptions = aspects.get(DESCRIPTIONS_ASPECT_KEY, {}).get('data', {})
  overview = aspects.get(OVERVIEW_ASPECT_KEY, {}).get('data', {})

  documentation = overview.get('content', '')
  has_documentation = not not documentation

  context = [
    _format_description(
      table_info['entrySource'].get('description', ''),
      descriptions.get('description', ''),
      blank_ok=False),
    '',
    'Schema:',
    '',
    _format_schema(schema, descriptions),
    '',
    'Documentation:',
    documentation if has_documentation else 'mo existing documentation',
  ]
  return '\n'.join(context), has_documentation


================================================
FILE: samples\enrichment\src\enrichment\metadata\snapshot.py
================================================
# Catalog metadata snapshot
#

import pathlib
import typing as t
import yaml

import google.cloud.dataplex_v1 as dataplex
import google.protobuf.field_mask_pb2 as field_mask_pb2
import google.protobuf.json_format as jsonpb

import enrichment.util.markdown as markdown


OVERVIEW_ASPECT_RESOURCE = 'projects/dataplex-types/locations/global/aspectTypes/overview'
OVERVIEW_ASPECT_KEY = '655216118709.global.overview'


def _entry_to_md(entry: t.Dict[str, t.Any]) -> t.Tuple[str, str]:
  table_name_parts = entry['entrySource']['resource'].split('/')
  table_name = f'{table_name_parts[1]}.{table_name_parts[3]}.{table_name_parts[5]}'

  aspects = entry.get('aspects', {})
  overview = aspects.get(OVERVIEW_ASPECT_KEY, {}).get('data', {})

  md = [
    f'---',
    f'name: {entry["name"]}',
    f'resource: {entry["entrySource"]["resource"]}',
    f'---',
    f'# {table_name}',
    f'## Overview\n',
    overview.get('content', '')
  ]

  return table_name, '\n'.join(md)


def _md_to_entry(md: str) -> t.Dict[str, t.Any]:
  frontmatter, body = markdown.parse(md)

  overview = ''
  if '## Overview' in body:
    overview = body.split('## Overview', 1)[1].strip()

  return {
    'name': frontmatter['name'],
    'aspects': {
      OVERVIEW_ASPECT_KEY: {
        'aspectType': OVERVIEW_ASPECT_RESOURCE,
        'data': {
          'content': overview,
          'contentType': 'MARKDOWN'
        }
      }
    },
    'entrySource': {
      'resource': frontmatter['resource']
    }
  }


def download_entries(dir: pathlib.Path, dataset: str):
  project_id, dataset_id = dataset.split('.')
  dataset_entry = None
  location_id = ''

  catalog = dataplex.CatalogServiceClient()

  search_entries_response = catalog.search_entries(
    request=dataplex.SearchEntriesRequest(
      name=f'projects/{project_id}/locations/global',
      scope=f'projects/{project_id}',
      query=f'type=dataset name={dataset_id}',
      page_size=1,
      semantic_search=True,
    )
  )
  for result in search_entries_response.results:
    dataset_entry = result.dataplex_entry
    location_id = dataset_entry.entry_source.location

  entry_filter = f'parent_entry="{dataset_entry.name}"'
  entry_group = f'projects/{project_id}/locations/{location_id}/entryGroups/@bigquery'

  list_entries_response = catalog.list_entries(
    request=dataplex.ListEntriesRequest(
      parent=entry_group,
      filter=entry_filter,
    )
  )

  for entry in list_entries_response.entries:
    entry = catalog.get_entry(
      request=dataplex.GetEntryRequest(
        name=entry.name,
        view='CUSTOM',
        aspect_types=[
          OVERVIEW_ASPECT_RESOURCE,
        ],
      )
    )

    table_name, markdown = _entry_to_md(jsonpb.MessageToDict(entry._pb))
    table_path = dir / f'{table_name}.md'
    table_path.write_text(markdown)
    print(f'Downloaded {table_name}')


def publish_entries(dir: pathlib.Path):
  catalog = dataplex.CatalogServiceClient()

  for table_path in dir.glob('*.md'):
    entry_data = _md_to_entry(table_path.read_text())

    updated_entry = dataplex.Entry()
    jsonpb.ParseDict(entry_data, updated_entry._pb, ignore_unknown_fields=True)

    catalog.update_entry(
      request=dataplex.UpdateEntryRequest(
        entry=updated_entry,
        update_mask=field_mask_pb2.FieldMask(paths=['aspects']),
        aspect_keys=[OVERVIEW_ASPECT_KEY],
      )
    )

    print(f'Published {table_path.stem}')


def list_entries(dir: pathlib.Path):
  return [table_path.stem for table_path in dir.glob('*.md')]

def update_entry(dir: pathlib.Path, out_dir: pathlib.Path, table_name: str, content: str):
  table_path = dir / f'{table_name}.md'
  out_table_path = out_dir / f'{table_name}.md'

  entry_data = _md_to_entry(table_path.read_text())
  entry_data['aspects'][OVERVIEW_ASPECT_KEY]['data']['content'] = content

  _, markdown = _entry_to_md(entry_data)
  out_table_path.write_text(markdown)


def show_entry(dir: pathlib.Path, table_name: str):
  table_path = dir / f'{table_name}.md'
  print(table_path.read_text())


================================================
FILE: samples\enrichment\src\enrichment\metadata\__init__.py
================================================


================================================
FILE: samples\enrichment\src\enrichment\publish.py
================================================
# Metadata publishing tool
#

import argparse
import pathlib
import sys

import enrichment.metadata.snapshot as snapshot


def main():
  parser = argparse.ArgumentParser(description='Publish metadata to the Catalog')
  parser.add_argument(
    '--dir',
    required=True,
    help='The directory containing the metadata to publish.',
  )
  args = parser.parse_args()

  dir = pathlib.Path(args.dir).resolve()
  if not dir.exists() or not dir.is_dir():
    print(f'Directory {dir} does not exist.')
    sys.exit(1)

  snapshot.publish_entries(dir)


if __name__ == '__main__':
  main()


================================================
FILE: samples\enrichment\src\enrichment\util\markdown.py
================================================
# Markdown Helpers
#

import typing as t
import yaml


def parse(md: str) -> t.Tuple[t.Dict[str, t.Any], str]:
  frontmatter = {}
  body = md

  if md.startswith('---\n'):
    parts = md.split('---\n', 2)
    if len(parts) >= 3:
      frontmatter = yaml.safe_load(parts[1]) or {}
      body = parts[2]

  return frontmatter, body


================================================
FILE: samples\enrichment\src\enrichment\util\__init__.py
================================================


================================================
FILE: samples\enrichment\src\enrichment\__init__.py
================================================


================================================
FILE: samples\enrichment\src\env.sh
================================================
if [ ! -d ".venv" ]; then
  python3 -m venv .venv
fi

source .venv/bin/activate

if [ "$1" == "--install" ]; then
  pip install -r requirements.txt
fi

export PYTHONDONTWRITEBYTECODE=1

export PYTHONWARNINGS="ignore::UserWarning:google.adk.features._feature_decorator"
export GOOGLE_GENAI_USE_VERTEXAI=True

export KC_ENRICH_SAMPLE_PROJECT=$(gcloud -q config get-value project)
if [ "$KC_ENRICH_SAMPLE_PROJECT" == "" ]; then
  echo "No sample project set. Please run 'gcloud config set project <project-id>'"
  exit 1
fi

export KC_ENRICH_SRC_DIR=$(cd -- "." && pwd)
export KC_ENRICH_SAMPLE_DOCS_DIR=$(cd -- "../sample/docs" && pwd)


================================================
FILE: samples\enrichment\src\requirements.txt
================================================
google-adk
google-auth
google-cloud-bigquery
google-cloud-dataplex
mcp[cli]
pyyaml


================================================
FILE: samples\enrichment\src\tools\fileskb\main.py
================================================
# MCP tools for a simple file-system based knowledge-base
#

import argparse
import os
import pathlib
import re
import sys
import typing as t
from mcp.server.fastmcp import FastMCP


mcp = FastMCP('fileskb')
KB_ROOT = pathlib.Path('.').resolve()


def _safe_path(path: str) -> pathlib.Path:
  '''Ensure the path is within the KB_ROOT to prevent directory traversal.
  '''
  clean_path = path.lstrip('/')
  full_path = (KB_ROOT / clean_path).resolve()
  if not str(full_path).startswith(str(KB_ROOT)):
    raise ValueError(f'Access denied: Path "{path}" is outside the knowledge base root.')
  return full_path


@mcp.tool(name='list')
def list_contents(path: str = '') -> str:
  '''List the contents of a directory in the knowledge base.
  
  Args:
    path: Optional relative path within the knowledge base to list.
  '''
  target_dir = _safe_path(path)
  if not target_dir or not target_dir.exists() or not target_dir.is_dir():
    return f'Path not found or is not a directory: {path}'

  lines = []
  for item in target_dir.iterdir():
    item_type = 'directory' if item.is_dir() else 'file'
    item_path = str(item.relative_to(KB_ROOT))
    lines.append(f'{item.name} | {item_path} | {item_type}')
  return '\n'.join(lines)


@mcp.tool(name='read')
def read_file(path: str) -> str:
  '''Read the contents of a file in the knowledge base.
  
  Args:
    path: Relative path to the file.
  '''
  target_file = _safe_path(path)
  if not target_file or not target_file.exists() or not target_file.is_file():
    return f'Path not found or is not a file: {path}'

  with open(target_file, 'r', encoding='utf-8') as f:
    return f.read()


@mcp.tool(name='search')
def search_content(query: str, path: str = '') -> t.List[t.Dict[str, t.Any]]:
  '''Search for a text query (regex supported) within markdown files.
  
  Args:
    query: The search string or regular expression.
    path: Optional relative path to restrict the search to a subdirectory or file.
  '''
  target_path = _safe_path(path)
  if not target_path or not target_path.exists():
    return f'Path not found: {path}'

  results = []
  try:
    pattern = re.compile(query, re.IGNORECASE)
  except re.error as e:
    return f'Invalid query: {e}'

  files_to_search = []
  if target_path.is_file():
    if target_path.suffix == '.md':
      files_to_search = [target_path]
  else:
    for root, _, files in os.walk(target_path):
      for file in files:
        if file.endswith('.md'):
          files_to_search.append(pathlib.Path(root) / file)

  for file_path in files_to_search:
    try:
      with open(file_path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
          if pattern.search(line):
            results.append({
              'file': str(file_path.relative_to(KB_ROOT)),
              'line': line_num,
              'content': line.strip()
            })
    except Exception:
      continue

  return results


if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='FileKB MCP Server')
  parser.add_argument(
    '--dir',
    required=True,
    help='Root directory for the knowledge base')
  args, unknown = parser.parse_known_args()

  KB_ROOT = pathlib.Path(args.dir).resolve()
  if not KB_ROOT.exists() or not KB_ROOT.is_dir():
    print(
      f'Error: Root directory "{KB_ROOT}" does not exist or is not a directory.',
      file=sys.stderr
    )
    sys.exit(1)

  mcp.run()


================================================
FILE: samples\enrichment\src\tools\fileskb\README.md
================================================
# FilesKB MCP Server

This is a simple MCP server for purposes of demonstrating how the enrichment
agent can be extended with MCP servers to access existing information sources.

The sample surfaces common file system capabilities - listing files, searching
files and reading files stored within a specified directory root.

## Using this MCP Server

The server can be provided to the agent using standard MCP configuration:

```json
{
  "mcpServers": {
    "fileskb": {
      "command": "/usr/local/google/home/nikhilko/p/kc/enrichment2/src/.venv/bin/python3",
      "args": [
        "/usr/local/google/home/nikhilko/p/kc/enrichment2/src/tools/fileskb/main.py",
        "--dir",
        "/usr/local/google/home/nikhilko/p/kc/enrichment2/demo/docs"
      ]
    }
  }
}
```


================================================
FILE: samples\README.md
================================================
# Samples

The samples demonstrate the use of Knowledge Catalog to manage metadata and
context to power agents.

## Discovery

Demonstrates building a search and discovery agent building on top of the
Search APIs offered by the catalog.

## Enrichment

Demonstrates an enrichment agent that can generate and enrich documentation
for assets managed in the catalog.


================================================
FILE: toolbox\enrichment\README.md
================================================
# Enrichment Agent

The enrichment agent for Knowledge Catalog provides a customizable agentic
workflow for extracting information from various sources to build metadata
about data assets, which can then be used as context.

## Usage

### Prerequisites

The enrichment agent depends on the [Metadata as Code](../mdcode/README.md) capability.
Follow the instructions on that page on using the `kcmd` tool.

### CLI

The package provides the `kcenrich` CLI tool. This is distributed as a standalone binary.

```bash
# Initialize a new catalog snapshot for a bigquery dataset
kcmd init --bigquery-dataset <projectId>.<datasetId>

# Pull the latest catalog snapshot from the Knowledge Catalog service
kcmd pull

# Run the enrichment tool
kcagent enrich --catalog-path . --tools-path tools --prompt-path prompt.md
```

## Developer Workflow

### Setup

```bash
git clone https://github.com/googlecloudplatform/knowledge-catalog
cd toolbox/enrichment
npm install
```

### Build

```bash
npm run build
```

### Test

```bash
npm run test
```

### Demo

The repository contains a self-contained demo. Running the demo involves creating a BigQuery dataset and a Dataplex EntryGroup within your cloud project.

**Initialize Environment**
```bash
export DEMO_CLOUD_PROJECT="<your-gcp-project-id>"
```

**Initialize gcloud**
```bash
gcloud auth application-default login
gcloud config set project $DEMO_CLOUD_PROJECT
gcloud config set compute/region us
```

**Setup demo resources**
```bash
bq query --use_legacy_sql=false <<EOF
CREATE SCHEMA IF NOT EXISTS \`${DEMO_CLOUD_PROJECT}.demo_ecommerce\`
OPTIONS (
  location = 'US',
  labels = [('usage', 'demo')]
);

CREATE TABLE IF NOT EXISTS \`${DEMO_CLOUD_PROJECT}.demo_ecommerce.events\`
PARTITION BY event_date_dt
AS
SELECT
  *,
  PARSE_DATE('%Y%m%d', event_date) AS event_date_dt
FROM
  \`bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*\`;
EOF
```

**Create and populate a catalog snapshot**
```bash
mkdir -p demo
cd demo
cat <<EOF > catalog.yaml
scope: bq-dataset.${DEMO_CLOUD_PROJECT}.demo_ecommerce

snapshot:
  entries:
    - dataplex-types.global.bigquery-dataset
    - dataplex-types.global.bigquery-table
  aspects:
    - dataplex-types.global.overview
EOF

../../mdcode/dist/kcmd pull
```

**Create and populate the tools**
```bash
cat <<EOF > prompt.md
Enrich the documentation of the assets using the internal organizational information.
Use the following sources:

* Fileset source
EOF
```
```bash
mkdir tools
cat <<EOF > tools/mcp.json
{
  "mcpServers": {
    "md-fileset": {
      "command": "../dist/md-fileset",
      "args": [ "--dir", "fileset" ]
    }
  }
}
EOF
```

```bash
mkdir -p tools/skills/fileset-source
cat <<EOF > tools/skills/fileset-source/SKILL.md
---
name: fileset-source
description: >
  Use the fileset source to find relevant markdown documents and extract information
  about assets.
---

The `md-fileset` mcp server provides the following tools to extract relevant
information from a directory hierarchy of markdown files:

* **list_fileset_contents** - browse and navigate the directory tree to list the
  contents of the specified path. The items may be files or sub-directories.

* **read_fileset_file** - read the contents of a file in the knowledge base. The entire
  contents are provided. Extract and summarise the relevant information based on
  the documentation being generated..

* **search_fileset_content** - searches the knowledge base and returns the matching files,
  along with matching line numbers and line snippets. This can be used to quickly
  find matches without having to list and read all files.

To work with a fileset effectively create search queries (use simple keyword queries
with individual tokens) to find relevant files, and then read the files to find relevant information. If one query does not work, try a few other keywords.
EOF
```

**Add Docs**  
Copy over the individual markdown files from [here](https://github.com/GoogleCloudPlatform/knowledge-catalog/tree/1e97103cdbf7e6425113a73304029ddb4f1f3a6b/samples/enrichment/sample/docs) into the `fileset/` directory

**Enrich the metadata**
```bash
../dist/kcagent enrich --catalog-path . --tools-path tools --prompt-path prompt.md
```

**Clean up**
```bash
bq rm -r ${DEMO_CLOUD_PROJECT}:demo-dataset
```


================================================
FILE: toolbox\enrichment\src\agent\enrich\agent.ts
================================================
// Agent implementation
//

import * as adk from '@google/adk';
import * as kcmd from 'kcmd';


const NAME = 'kcagent-enrich';
const MODEL = 'gemini-2.5-flash';
const DESCRIPTION = 'Enriches the metadata in knowledge catalog.';
const PROMPT = `
Your are an experienced data steward and data analyst.

Your task is build comprehensive documentation about the specified data asset
based on the provided sources of information.

The documentation should be in the form of markdown readme consisting of a
top-level summary organized into a few paragraphs, and then a list of sections
and bullet points for more specific details.

The documentation should optimize for both humans and agents, to aid in
improving discoverability of the asset via search, improved understanding of
the data in the asset, and how it should be used in a business context to
answer data analytics queries.

Good Documentation:

* Comprehensive, yet, compact and easy to read. Organized into few paragraphs
  followeby succint lists with clear section headings.

* Captures key details that can help find the data asset, understand its data
  and how to use it.

* Does not simply duplicate information already present in the metadata or directly
  copy information verbatim. Always syntheisize to ensure the documentation can
  be consumed by itself.

* Only uses the asset metadata and information found about it. Avoids ambiguous
  statements (such as 'may', 'likely', 'could' etc). Use clear statements grounded
  in facts.

* If there is no useful information, it is acceptable to indicate that.

* Include a list of citations.

Documentation Template:

A few paragraphs of key information summarized for easy read and search.

## Data Details

* List of most salient details about the data

## Usage Details

* List of most salient details about how to use the data

## Citations

* List of sources (formatted as title, reference)


Workflow:

1. Understand the existing metadata of the asset, such as its name, description
   schema, etc. and existing documentation.

2. Use the available tools to gather more information about the asset and any
   usage details. In order to do so, formulate search queries to search the
   information sources, retrieve relevant content. Extract relevant details from
   those, and update the documentation with meaningful net new information
   while preserving the relevant bits of existing information. Update the list
   of citations to record the sources of information. Make sure all documentation
   content is based on actual information. Avoid making up information.

   If no additional information was found, and there should not be any documentation
   updates. Simply output that no information was found and documentation was not
   updated. Skip the following steps.

3. Review and revise the generated documentation to ensure it is succinct, and easy
   to consume, and clear, and uses unambiguous language.

4. Invoke the 'update_documentation' tool with the updated documentation.
   Finally, output a message saying the documentation has been updated.
`;

const apiContext = kcmd.gcp.ApiContext.default();

export function createAgent(tools: adk.ToolUnion[]): adk.Agent {
  return new adk.Agent({
    name: NAME,
    description: DESCRIPTION,
    instruction: PROMPT,
    tools: tools,
    model: new adk.Gemini({
      model: MODEL,
      vertexai: true,
      project: apiContext.project,
      location: apiContext.location,
    }),
    generateContentConfig: {
      thinkingConfig: {
        includeThoughts: true,
        thinkingBudget: -1,
      },
    },
  });
}


================================================
FILE: toolbox\enrichment\src\agent\enrich\command.ts
================================================
// Enrichment process
//

import * as fs from 'node:fs';
import * as adk from '@google/adk';
import * as kcmd from 'kcmd';
import { createAgent } from './agent.js';
import { loadMcpTools, loadSkills } from '../tools';

export interface EnrichOptions {
  catalogPath: string;
  toolsPath: string;
  promptPath: string;
}

export async function enrichCommand(options: EnrichOptions) {
  console.log('Enriching metadata...');

  let stat = await fs.promises.stat(options.catalogPath);
  if (!stat.isDirectory()) {
    console.error(`Error: ${options.catalogPath} is not a directory.`);
    process.exit(1);
  }
  stat = await fs.promises.stat(options.toolsPath);
  if (!stat.isDirectory()) {
    console.error(`Error: ${options.toolsPath} is not a directory.`);
    process.exit(1);
  }
  stat = await fs.promises.stat(options.promptPath);
  if (!stat.isFile()) {
    console.error(`Error: ${options.promptPath} is not a file.`);
    process.exit(1);
  }

  const commonPrompt = await fs.promises.readFile(options.promptPath, 'utf-8');
  const mcpTools = await loadMcpTools(options.toolsPath);
  const skills = await loadSkills(options.toolsPath);

  const context = kcmd.gcp.ApiContext.default();
  const catalog = await kcmd.CatalogSnapshot.fromPath(options.catalogPath, context);

  const entryNames = await catalog.listEntries();
  for (const name of entryNames) {
    console.log(`Processing: ${name}`);
    const entry = await catalog.lookupEntry(name);
    if (!entry) {
      console.error(`Error: Entry ${name} not found.`);
      continue;
    }

    const updateDocumentationTool = new adk.FunctionTool({
      name: 'update_documentation',
      description: 'Update the documentation of an asset',
      parameters: {
        type: 'OBJECT' as any,
        properties: {
          documentation: {
            type: 'STRING' as any,
            description: 'The new documentation content (Markdown format)',
          },
        },
        required: ['documentation'],
      },
      execute: async (input: any) => {
        console.log('Update tool called;')
        const documentation = input.documentation as string;
        entry.aspects = { ...(entry.aspects ?? {}) };
        entry.aspects['dataplex-types.global.overview'] = {
          content: documentation,
          contentType: 'MARKDOWN'
        };
        await catalog.updateEntry(entry, ['dataplex-types.global.overview']);
        return { result: 'Successfully updated documentation' };
      },
    });

    const agent = createAgent([updateDocumentationTool, ...mcpTools, ...skills]);
    const runner = new adk.InMemoryRunner({
      agent: agent,
      appName: 'kcagent'
    });

    const events = runner.runEphemeral({
      userId: 'cli-user',
      newMessage: {
        role: 'user',
        parts: [{ text: createPrompt(entry, commonPrompt) }]
      }
    });

    for await (const event of events) {
      const structuredEvents = adk.toStructuredEvents(event);
      for (const se of structuredEvents) {
        if (se.type === adk.EventType.THOUGHT) {
          if (se.content?.trim()) {
            console.log(`\x1b[90m[Thought]: ${se.content.trim()}\x1b[0m`);
          }
        }
        else if (se.type === adk.EventType.CONTENT) {
          if (se.content?.trim()) {
            console.log(`\x1b[1;30m[Agent] ${se.content.trim()}\x1b[0m`);
          }
        }
        else if (se.type === adk.EventType.TOOL_CALL) {
          console.log(`\x1b[30m[Tool Invoke] ${se.call.name}\n${JSON.stringify(se.call.args || {})}\x1b[0m`);
        }
        else if (se.type === adk.EventType.TOOL_RESULT) {
          console.log(`\x1b[30m[Tool Result] ${se.result.name}\n${JSON.stringify(se.result.response || {})}\x1b[0m`);
        }
      }
    }
  }
}


function createPrompt(entry: kcmd.Entry, commonPrompt: string): string {
  const lines: string[] = [];

  lines.push(`Asset: ${entry.name}`);

  const schema = entry.aspects?.['dataplex-types.global.schema'];
  if (schema) {
    lines.push('Schema:');
    lines.push(JSON.stringify(schema.fields, null, 2));
  }

  const overview = entry.aspects?.['dataplex-types.global.overview'];
  if (overview) {
    lines.push('Existing Documentation');
    lines.push(overview.content);
  }
  else {
    lines.push('No existing documentation found');
  }

  lines.push('');
  lines.push(commonPrompt);

  return lines.join('\n');
}


================================================
FILE: toolbox\enrichment\src\agent\main.ts
================================================
// Application entrypoint
//

// Ensure this happens first, before anything else is loaded.
import './utils/patchpb';
import './utils/patchadk';

import * as adk from '@google/adk';
import * as cac from 'cac';
import { enrichCommand } from './enrich/command.js';

// Suppress the overly noisy logging from the ADK
adk.setLogLevel(adk.LogLevel.WARN);

const cli = cac.cac('kcagent').version('1.0.0').help();
cli.command('enrich', 'Enrich the metadata catalog')
   .option('--catalog-path <path>', 'Path to the metadata catalog')
   .option('--tools-path <path>', 'Path to the config directory')
   .option('--prompt-path <path>', 'Path to the prompt file')
   .action(enrichCommand);


function main() {
  cli.parse();

  if (!cli.matchedCommand) {
    if (cli.args.length > 0) {
      console.error(`Error: Unknown command '${cli.args[0]}'`);
    }

    cli.outputHelp();
    process.exit(1);
  }
}


main();


================================================
FILE: toolbox\enrichment\src\agent\tools.ts
================================================
// Tools and Skills Loader
//

import * as fs from 'node:fs';
import * as path from 'node:path';
import * as adk from '@google/adk';


function expandEnvVars(str: string): string {
  return str.replace(/\$(\w+)|\${(\w+)}/g, (_, m1, m2) => {
    const varName = m1 || m2;
    return process.env[varName] || '';
  });
}

export async function loadMcpTools(configPath: string): Promise<adk.MCPToolset[]> {
  const mcpConfigPath = path.join(configPath, 'mcp.json');
  const stat = await fs.promises.stat(mcpConfigPath);
  if (!stat.isFile()) {
    return [];
  }

  try {
    const content = await fs.promises.readFile(mcpConfigPath, 'utf-8');
    const config = JSON.parse(content);
    const toolsets: adk.MCPToolset[] = [];
    const mcpServers = config.mcpServers || {};

    for (const [_, serverConfig] of Object.entries(mcpServers) as [string, any][]) {
      if (serverConfig.command) {
        const command = expandEnvVars(serverConfig.command);
        const args = (serverConfig.args || []).map((arg: string) => expandEnvVars(arg));
        const env = serverConfig.env
          ? Object.fromEntries(
            Object.entries(serverConfig.env).map(([k, v]) => [k, expandEnvVars(String(v))])
          )
          : undefined;
        const timeout = typeof serverConfig.timeout === 'number' ? serverConfig.timeout : undefined;

        toolsets.push(
          new adk.MCPToolset({
            type: 'StdioConnectionParams',
            serverParams: { command, args, env },
            timeout
          })
        );
      }
      else if (serverConfig.httpUrl) {
        const url = expandEnvVars(serverConfig.httpUrl);
        const timeout = typeof serverConfig.timeout === 'number' ? serverConfig.timeout : undefined;

        toolsets.push(
          new adk.MCPToolset({
            type: 'StreamableHTTPConnectionParams',
            url,
            timeout
          })
        );
      }
    }
    return toolsets;
  }
  catch (error: any) {
    console.warn(`Warning: Failed to load mcp.json: ${error.message}`);
    return [];
  }
}

export async function loadSkills(configPath: string): Promise<adk.SkillToolset[]> {
  const skillsBasePath = path.join(configPath, 'skills');
  const stat = await fs.promises.stat(skillsBasePath);
  if (!stat.isDirectory()) {
    return [];
  }

  try {
    const skillsMap = await adk.loadAllSkillsInDir(skillsBasePath);
    if (Object.keys(skillsMap).length === 0) {
      return [];
    }

    return [
      new adk.SkillToolset(skillsMap, {
        codeExecutor: new adk.UnsafeLocalCodeExecutor()
      })
    ];
  }
  catch (error: any) {
    console.warn(`Warning: Failed to load skills: ${error.message}`);
    return [];
  }
}


================================================
FILE: toolbox\enrichment\src\agent\utils\patchadk.ts
================================================
// ADK Patches for MCP Toolset and Runner
// https://github.com/google/adk-js/issues/377
//

import * as adk from '@google/adk';

// Track all active MCP sessions globally to allow automatic failsafe runner clean-up
const activeMcpSessions = new Set<any>();

// Track MCP client sessions per MCPSessionManager using a WeakMap for targeted toolset clean-up
const mcpManagerSessions = new WeakMap<any, any[]>();

const originalCreateSession = adk.MCPSessionManager.prototype.createSession;
adk.MCPSessionManager.prototype.createSession = async function (this: any) {
  const session = await originalCreateSession.call(this);

  // Record globally for automatic runner clean-up
  activeMcpSessions.add(session);

  // Record per-manager for targeted toolset close
  let sessions = mcpManagerSessions.get(this);
  if (!sessions) {
    sessions = [];
    mcpManagerSessions.set(this, sessions);
  }
  sessions.push(session);

  return session;
};

async function cleanUpMcpSessions(): Promise<void> {
  for (const session of activeMcpSessions) {
    try {
      await session.close();
    }
    catch (error) {
      // Ignore close errors
    }
  }
  activeMcpSessions.clear();
}

// Patch MCPToolset's close() method to support manual closing
(adk as any).MCPToolset.prototype.close = async function (this: any): Promise<void> {
  // MCPToolset extends BaseToolset which requires the close() method to
  // release resources. Currently, the framework leaves it empty. If the
  // framework is updated to call .close() automatically inside agents, or
  // if a developer decides to manually close the tools as they did before,
  // calling toolset.close() will now actually clean up the background child
  // processes.
  const sessions = mcpManagerSessions.get(this.mcpSessionManager);
  if (sessions) {
    for (const session of sessions) {
      try {
        await session.close();
        activeMcpSessions.delete(session);
      }
      catch (error) {
        // Ignore close errors
      }
    }
    sessions.length = 0;
  }
};

// Patch adk.Runner to automatically clean up sessions when execution completes
const originalRunEphemeral = (adk as any).Runner.prototype.runEphemeral;
(adk as any).Runner.prototype.runEphemeral = function (this: any, ...args: any[]) {
  // The ADK Runner currently does not call close() on its agent's toolsets
  // when an execution turn completes. Because of this framework limitation,
  // even if the close() method on MCPToolset is implemented correctly, the
  // sessions would still leak unless the developer explicitly writes a finally
  // block in their CLI code to loop through all tools and close them.

  const generator = originalRunEphemeral.apply(this, args);
  return (async function* () {
    try {
      yield* generator;
    }
    finally {
      await cleanUpMcpSessions();
    }
  })();
};

const originalRunAsync = (adk as any).Runner.prototype.runAsync;
(adk as any).Runner.prototype.runAsync = function (this: any, ...args: any[]) {
  // The ADK Runner currently does not call close() on its agent's toolsets
  // when an execution turn completes. Because of this framework limitation,
  // even if the close() method on MCPToolset is implemented correctly, the
  // sessions would still leak unless the developer explicitly writes a finally
  // block in their CLI code to loop through all tools and close them.

  const generator = originalRunAsync.apply(this, args);
  return (async function* () {
    try {
      yield* generator;
    }
    finally {
      await cleanUpMcpSessions();
    }
  })();
};


================================================
FILE: toolbox\enrichment\src\agent\utils\patchpb.ts
================================================
// Patch Protobuf to make it work with static bundlers.
// Ensure this is loaded before any other modules are loaded.
//
// When compiling a standalone binary using `bun build --compile`, Bun's static analyzer
// misses protobufjs's dynamic `require("long")` calls. This module explicitly imports
// and registers `long` globally so that protobufjs successfully resolves 64-bit integers
// at runtime. Protobuf is a dependency of the Google ADK.
//
// https://github.com/protobufjs/protobuf.js/issues/2285

import Long from 'long';
import * as protobufMinimal from 'protobufjs/minimal.js';
import * as protobuf from 'protobufjs';

(globalThis as any).Long = Long;
protobufMinimal.util.Long = Long as any;
protobufMinimal.configure();
protobuf.util.Long = Long as any;
protobuf.configure();


================================================
FILE: toolbox\enrichment\src\tools\md\fileset.ts
================================================
// Markdown fileset collection implementation
//

import * as fs from 'fs';
import * as path from 'path';

export interface SearchResult {
  file: string;
  line: number;
  content: string;
}

export class MarkdownFileset {
  private readonly root: string;

  constructor(root: string) {
    this.root = path.resolve(root);
    if (!fs.existsSync(this.root) || !fs.statSync(this.root).isDirectory()) {
      throw new Error(`Root directory "${root}" does not exist or is not a directory.`);
    }
  }

  private safePath(relativePath: string): string {
    const fullPath = path.resolve(this.root, relativePath.replace(/^\/+/, ''));

    const isInside = fullPath === this.root || fullPath.startsWith(this.root + path.sep);
    if (!isInside) {
      throw new Error(`Access denied: Path "${relativePath}" is invalid.`);
    }
    return fullPath;
  }

  async listContents(relativePath: string = ''): Promise<string> {
    const targetDir = this.safePath(relativePath);
    if (!fs.existsSync(targetDir) || !fs.statSync(targetDir).isDirectory()) {
      return `Path not found or is not a directory: ${relativePath}`;
    }

    const items = await fs.promises.readdir(targetDir);
    const lines: string[] = [];
    for (const name of items) {
      const fullPath = path.join(targetDir, name);
      const stat = await fs.promises.stat(fullPath);
      const itemType = stat.isDirectory() ? 'directory' : 'file';
      const itemPath = path.relative(this.root, fullPath);
      lines.push(`${name} | ${itemPath} | ${itemType}`);
    }
    return lines.join('\n');
  }

  async readFile(relativePath: string): Promise<string> {
    const targetFile = this.safePath(relativePath);
    if (!fs.existsSync(targetFile) || !fs.statSync(targetFile).isFile()) {
      return `Path not found or is not a file: ${relativePath}`;
    }
    return await fs.promises.readFile(targetFile, 'utf-8');
  }

  private async getMarkdownFiles(dir: string): Promise<string[]> {
    const files: string[] = [];
    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
    for (const entry of entries) {
      const fullPath = path.join(dir, entry.name);
      if (entry.isDirectory()) {
        const subFiles = await this.getMarkdownFiles(fullPath);
        files.push(...subFiles);
      }
      else if (entry.isFile() && entry.name.endsWith('.md')) {
        files.push(fullPath);
      }
    }
    return files;
  }

  async searchContents(query: string, relativePath: string = ''): Promise<SearchResult[] | string> {
    const targetPath = this.safePath(relativePath);
    if (!fs.existsSync(targetPath)) {
      return `Path not found: ${relativePath}`;
    }

    let regex: RegExp;
    try {
      regex = new RegExp(query, 'i');
    }
    catch (error: any) {
      return `Invalid query: ${error.message}`;
    }

    const filesToSearch: string[] = [];
    const stat = await fs.promises.stat(targetPath);
    if (stat.isFile()) {
      if (targetPath.endsWith('.md')) {
        filesToSearch.push(targetPath);
      }
    }
    else if (stat.isDirectory()) {
      const walkFiles = await this.getMarkdownFiles(targetPath);
      filesToSearch.push(...walkFiles);
    }

    const results: SearchResult[] = [];

    for (const file of filesToSearch) {
      try {
        const content = await fs.promises.readFile(file, 'utf-8');
        const lines = content.split(/\r?\n/);
        for (let i = 0; i < lines.length; i++) {
          const line = lines[i];
          if (regex.test(line)) {
            results.push({
              file: path.relative(this.root, file),
              line: i + 1,
              content: line.trim()
            });
          }
        }
      }
      catch {
        continue;
      }
    }

    return results;
  }
}


================================================
FILE: toolbox\enrichment\src\tools\md\main.ts
================================================
// MCP Application Entrypoint for Markdown fileset tool
//

import * as fs from 'node:fs';
import * as path from 'node:path';
import * as cac from 'cac';
import { MarkdownFileset } from './fileset';
import { runServer } from './server';


async function main() {
  const cli = cac.cac('md-fileset').version('1.0.0').help();
  cli.option('--dir <dir>', 'Root directory for the markdown fileset');

  const parsed = cli.parse(process.argv);
  if (parsed.options.help || parsed.options.version) {
    return;
  }
  if (!parsed.options.dir) {
    console.error('Error: --dir option is required');
    cli.outputHelp();
    process.exit(1);
  }

  const root = path.resolve(parsed.options.dir);
  if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
    console.error(`Error: Root directory "${root}" does not exist or is not a directory.`);
    process.exit(1);
  }

  const fileset = new MarkdownFileset(root);
  await runServer(fileset);
}

main();


================================================
FILE: toolbox\enrichment\src\tools\md\server.ts
================================================
// MCP Server implementation
//

import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { z } from 'zod';
import { MarkdownFileset } from './fileset';

async function handleToolCall(action: () => Promise<any>) {
  try {
    const result = await action();
    const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
    return {
      content: [{ type: 'text' as const, text }]
    };
  }
  catch (error: any) {
    return {
      isError: true,
      content: [{ type: 'text' as const, text: error.message }]
    };
  }
}

export async function runServer(fileset: MarkdownFileset) {
  const server = new McpServer({
    name: 'md-fileset',
    version: '1.0.0'
  });

  server.registerTool('list_fileset_contents', {
      description: 'List the markdown documents in the fileset.',
      inputSchema: {
        path: z.string().optional().default('')
               .describe('Optional relative path within the markdown fileset to list.'),
      },
    },
    async ({ path: relativePath = '' }) => {
      return handleToolCall(() => fileset.listContents(relativePath));
    });

  server.registerTool('search_fileset_contents', {
      description: 'Search for a text query (regex supported) within the markdown files.',
      inputSchema: {
        query: z.string().describe('The search string or regular expression.'),
        path: z.string().optional().default('')
               .describe('Optional relative path of the sub directory to search within.'),
      },
    },
    async ({ query, path: relativePath = '' }) => {
      return handleToolCall(() => fileset.searchContents(query, relativePath));
    });

  server.registerTool('read_fileset_file', {
      description: 'Read the contents of a file in the markdown fileset.',
      inputSchema: {
        path: z.string().describe('Relative path to the file.'),
      },
    },
    async ({ path: relativePath }) => {
      return handleToolCall(() => fileset.readFile(relativePath));
    });

  const transport = new StdioServerTransport();
  await server.connect(transport);
}


================================================
FILE: toolbox\mdcode\demo\bq\cleanup.ts
================================================
import * as cp from 'child_process';
import * as kcmd from 'kcmd';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const dataset = `${project}.demo_ecommerce`;
const sql = `
DROP SCHEMA IF EXISTS \`${dataset}\` CASCADE;
`;

cp.execSync('bq query --use_legacy_sql=false', {
  encoding: 'utf8',
  input: sql,
  stdio: 'inherit',
});
console.log(`Deleted demo BigQuery resources in dataset ${dataset}`);
console.log();


================================================
FILE: toolbox\mdcode\demo\bq\setup.ts
================================================
import * as cp from 'child_process';
import * as path from 'node:path';
import * as kcmd from 'kcmd';
import { YAML } from 'bun';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const dataset = `${project}.demo_ecommerce`;
const sql = `
CREATE SCHEMA IF NOT EXISTS \`${dataset}\`
OPTIONS (
  location = 'US',
  labels = [('usage', 'demo')]
);


CREATE TABLE IF NOT EXISTS \`${dataset}.events\`
PARTITION BY event_date_dt
AS
SELECT
  *,
  PARSE_DATE('%Y%m%d', event_date) AS event_date_dt
FROM
  \`bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*\`;
`;

cp.execSync('bq query --use_legacy_sql=false', {
  encoding: 'utf8',
  input: sql,
  stdio: 'inherit',
});
console.log(`Created demo BigQuery resources in dataset ${dataset}`);
console.log();


await Bun.file(path.join(process.cwd(), 'catalog.yaml')).write(YAML.stringify({
  scope: `bq-dataset.${dataset}`,
  snapshot: {
    entries: [
      'dataplex-types.global.bigquery-dataset',
      'dataplex-types.global.bigquery-table',
      'dataplex-types.global.bigquery-view'
    ],
    aspects: [
      'dataplex-types.global.overview'
    ]
  },
  publishing: {
    entries: [
      'dataplex-types.global.bigquery-dataset',
      'dataplex-types.global.bigquery-table',
      'dataplex-types.global.bigquery-view'
    ],
    aspects: [
      'dataplex-types.global.overview'
    ]
  }
}, null, 2));
console.log('Created catalog.yaml manifest');


================================================
FILE: toolbox\mdcode\demo\bq\update.ts
================================================
import { YAML } from 'bun';
import path from 'node:path';

const filePath = path.join(process.cwd(), process.argv[2]);
const file = Bun.file(filePath);
const content = await file.text();
const metadata = YAML.parse(content) as Record<string, any>;

metadata['aspects']['dataplex-types.global.overview'] = {
  content: 'sample updated documentation',
  contentType: 'MARKDOWN'
}

const updatedContent = YAML.stringify(metadata, null, 2);
file.write(updatedContent);


================================================
FILE: toolbox\mdcode\demo\kb\cleanup.ts
================================================
import * as cp from 'child_process';
import * as kcmd from 'kcmd';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const location = context.location;
const entryGroup = 'demo_kb';

function dataplex(cmd: string, data: string|null=null) {
  cmd = 'gcloud -q dataplex ' + cmd + ` --project ${project} --location ${location}`;
  cp.execSync(cmd, { encoding: 'utf8', input: data ?? undefined, stdio: 'inherit'});
}

dataplex(`entry-groups delete ${entryGroup}`);
console.log('Deleted catalog knowledge base entries');
console.log();



================================================
FILE: toolbox\mdcode\demo\kb\setup.ts
================================================
import * as cp from 'child_process';
import * as path from 'node:path';
import * as kcmd from 'kcmd';
import { YAML } from 'bun';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const location = context.location;
const entryGroup = 'demo_kb';
const genericResource = 'projects/dataplex-types/locations/global/entryTypes/generic';

function dataplex(cmd: string, data: string|null=null) {
  cmd = 'gcloud dataplex ' + cmd + ` --project ${project} --location ${location}`;
  cp.execSync(cmd, { encoding: 'utf8', input: data ?? undefined, stdio: 'inherit'});
}

function createEntry(name: string, displayName: string, description: string) {
  dataplex(`entries create --entry-group ${entryGroup} ` +
           `--entry-type ${genericResource} ` +
           `--entry-source-display-name "${displayName}" ` +
           `--entry-source-description "${description}" ` +
           `--entry-source-labels usage=demo,sample=true ` +
           `--aspects /tmp/aspects.json ` +
           name)
}

const aspectsFile = Bun.file('/tmp/aspects.json');
await aspectsFile.write(JSON.stringify({
  'dataplex-types.global.generic': {
    aspectType: 'dataplex-types.global.generic',
    data: {}
  },
  'dataplex-types.global.overview': {
    aspectType: 'dataplex-types.global.overview',
    data: {
      content: '# Placeholder Content\n\nLorem ipsum dolor met',
      contentType: 'MARKDOWN'
    }
  }
}));

try {
  dataplex(`entry-groups create ${entryGroup}`);
  createEntry('index', 'Index', 'Demo knowledge base index of top-level sections');
  createEntry('tags/tag1', 'Tag 1', 'First tag');
  createEntry('tags/tag2', 'Tag 2', 'Second tag');
  createEntry('topic1', 'Topic 1', 'Initial placeholder topic');
  createEntry('dir/index', 'Index', 'Some contents in a directory');
  createEntry('dir/topic1', 'Topic 1', 'Some topic 1');
  createEntry('dir/topic2', 'Topic 2', 'Some topic 2');
  createEntry('dir/topic3', 'Topic 3', 'Some topic 3');

  console.log('Created catalog knowledge base entries');
  console.log();
}
catch {
  // Might already exist
}

await Bun.file(path.join(process.cwd(), 'catalog.yaml')).write(YAML.stringify({
  scope: `kb.${project}.${location}.${entryGroup}`,
  snapshot: {
    entries: [
      'dataplex-types.global.generic'
    ],
    aspects: [
      'dataplex-types.global.overview'
    ]
  },
  publishing: {
    entries: [
      'dataplex-types.global.generic'
    ],
    aspects: [
      'dataplex-types.global.overview'
    ]
  }
}, null, 2));
console.log('Created catalog.yaml manifest');


================================================
FILE: toolbox\mdcode\demo\kb\update.ts
================================================
import * as kcmd from 'kcmd';

const context = kcmd.gcp.ApiContext.default();
const catalogSnapshot = await kcmd.CatalogSnapshot.fromPath('.', context);

const entry = await catalogSnapshot.lookupEntry('index');
if (!entry.aspects) {
  entry.aspects = {};
}
entry.aspects['dataplex-types.global.overview'] = {
  content: 'New updated content\n',
  contentType: 'MARKDOWN'
};
await catalogSnapshot.updateEntry(entry, ['dataplex-types.global.overview']);


================================================
FILE: toolbox\mdcode\demo\okf\catalog\datasets\ga4_obfuscated_sample_ecommerce.md
================================================
---
type: BigQuery Dataset
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce
title: BigQuery sample dataset for Google Analytics ecommerce web implementation
description: A sample of obfuscated Google Analytics BigQuery event export data for
  three months from the Google Merchandise Store is available as a public dataset
  in BigQuery.
tags:
- ecommerce
- web analytics
- Google Analytics
- BigQuery
- public dataset
timestamp: '2026-05-28T22:49:59+00:00'
---

# Overview
The `ga4_obfuscated_sample_ecommerce` dataset contains obfuscated Google Analytics BigQuery event export data for three months (November 2020 to January 2021) from the Google Merchandise Store. This public dataset is available in BigQuery and emulates a real-world dataset.

# Pre-requisites
To work with this dataset, you need access to a Google Cloud project with the BigQuery API enabled. You can use BigQuery Sandbox mode or the Free usage tier for exploration and sample queries.

# Limitations
The dataset contains obfuscated data with placeholder values like `<Other>`, `NULL`, and `''`. Due to obfuscation, the internal consistency of the dataset might be somewhat limited. It cannot be compared to the Google Analytics Demo Account.

# Using the dataset
You can access the `ga4_obfuscated_sample_ecommerce` dataset via the BigQuery UI in the Cloud Console.

## Sample Query
The following query shows the number of unique events, users, and days in the dataset:

```sql
SELECT
  COUNT(*) AS event_count,
  COUNT(DISTINCT user_pseudo_id) AS user_count,
  COUNT(DISTINCT event_date) AS day_count
FROM `bigquery-public-data.ga4_obfuscated_sample_ecommerce.events_*`
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: toolbox\mdcode\demo\okf\catalog\datasets\index.md
================================================
# BigQuery Dataset

* [BigQuery sample dataset for Google Analytics ecommerce web implementation](ga4_obfuscated_sample_ecommerce.md) - A sample of obfuscated Google Analytics BigQuery event export data for three months from the Google Merchandise Store is available as a public dataset in BigQuery.


================================================
FILE: toolbox\mdcode\demo\okf\catalog\index.md
================================================
# Subdirectories

* [datasets](datasets/index.md) - A sample of obfuscated Google Analytics BigQuery event export data for three months from the Google Merchandise Store is available as a public dataset in BigQuery.
* [references](references/index.md) - This directory contains specifications for data joins and definitions for user activity and purchase metrics.
* [tables](tables/index.md) - Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\index.md
================================================
# Subdirectories

* [joins](joins/index.md) - Join Google Analytics event data with Google Ads click data.
* [metrics](metrics/index.md) - This directory contains definitions for metrics related to user activity, events, and purchase behavior.


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\joins\events___ads_clickstats.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Join Google Analytics Events to Google Ads Clicks
description: Join Google Analytics event data with Google Ads click data.
tags:
- join
- Google Ads
timestamp: '2026-05-28T22:51:46+00:00'
---

Join Google Analytics event data with Google Ads click data.

```sql
GA_EVENTS.collected_traffic_source.gclid = ADS_CLICKS.gclid
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\joins\index.md
================================================
# Reference

* [Join Google Analytics Events to Google Ads Clicks](events___ads_clickstats.md) - Join Google Analytics event data with Google Ads click data.


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\avg_pageviews.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Average Pageviews
description: The average number of pageviews per user.
tags:
- metric
timestamp: '2026-05-28T22:51:43+00:00'
---

The average number of pageviews per user.

```sql
SUM(page_view_count) / COUNT(*)
-- where page_view_count is COUNTIF(event_name = 'page_view') per user
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\avg_spend_per_purchase_session_by_user.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/advanced-queries
title: Average Spend Per Purchase Session By User
description: The average amount of money spent per purchase session for each individual
  user.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:52:29+00:00'
---

The average amount of money spent per purchase session for each individual user.

```sql
AVG(total_session_spend)
-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id
```

# Citations
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\avg_transactions_per_purchaser.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: Average Transactions Per Purchaser
description: The average number of transactions made by purchasers.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:51:41+00:00'
---

The average number of transactions made by purchasers.

```sql
COUNT(*) / COUNT(DISTINCT user_pseudo_id)
-- for events where event_name IN ('in_app_purchase', 'purchase')
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\day_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: Day Count
description: Total number of unique days.
tags:
- metric
timestamp: '2026-05-28T22:50:10+00:00'
---

Total number of unique days.

```sql
COUNT(DISTINCT event_date)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\event_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: Event Count
description: Total number of events.
tags:
- metric
timestamp: '2026-05-28T22:50:07+00:00'
---

Total number of events.

```sql
COUNT(*)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\index.md
================================================
# Reference

* [Average Pageviews](avg_pageviews.md) - The average number of pageviews per user.
* [Average Spend Per Purchase Session By User](avg_spend_per_purchase_session_by_user.md) - The average amount of money spent per purchase session for each individual user.
* [Average Transactions Per Purchaser](avg_transactions_per_purchaser.md) - The average number of transactions made by purchasers.
* [Day Count](day_count.md) - Total number of unique days.
* [Event Count](event_count.md) - Total number of events.
* [New User Count](new_user_count.md) - The number of unique users who triggered a first_visit or first_open event.
* [Overall Average Spend Per Purchase Session](overall_avg_spend_per_purchase_session.md) - The overall average amount spent across all unique purchase sessions.
* [User Count](user_count.md) - Total number of unique users.


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\new_user_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/basic-queries
title: New User Count
description: The number of unique users who triggered a first_visit or first_open
  event.
tags:
- metric
timestamp: '2026-05-28T22:51:38+00:00'
---

The number of unique users who triggered a `first_visit` or `first_open` event.

```sql
SUM(is_new_user)
-- where is_new_user is MAX(IF(event_name IN ('first_visit', 'first_open'), 1, 0)) grouped by user_pseudo_id
```

# Citations
- https://developers.google.com/analytics/bigquery/basic-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\overall_avg_spend_per_purchase_session.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/advanced-queries
title: Overall Average Spend Per Purchase Session
description: The overall average amount spent across all unique purchase sessions.
tags:
- metric
- ecommerce
timestamp: '2026-05-28T22:52:32+00:00'
---

The overall average amount spent across all unique purchase sessions.

```sql
AVG(total_session_spend)
-- where total_session_spend is SUM(COALESCE(...)) for event_name = 'purchase' events within a session, grouped by user_pseudo_id and ga_session_id
```

# Citations
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\references\metrics\user_count.md
================================================
---
type: Reference
resource: https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
title: User Count
description: Total number of unique users.
tags:
- metric
timestamp: '2026-05-28T22:50:09+00:00'
---

Total number of unique users.

```sql
COUNT(DISTINCT user_pseudo_id)
```

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset


================================================
FILE: toolbox\mdcode\demo\okf\catalog\tables\events_.md
================================================
---
type: BigQuery Table
resource: https://bigquery.googleapis.com/v2/projects/bigquery-public-data/datasets/ga4_obfuscated_sample_ecommerce/tables/events_*
title: Events table (Google Analytics BigQuery Export)
description: Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce`
  dataset.
tags:
- events
- Google Analytics
- BigQuery
- ecommerce
- schema
- basic queries
- advanced queries
timestamp: '2026-05-28T22:53:05+00:00'
---

# Overview
The `events_` table is a sharded BigQuery table containing Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.

# Metrics
- [Event Count](../references/metrics/event_count.md) — Total number of events.
- [User Count](../references/metrics/user_count.md) — Total number of unique users.
- [Day Count](../references/metrics/day_count.md) — Total number of unique days.
- [New User Count](../references/metrics/new_user_count.md) — The number of unique users who triggered a first_visit or first_open event.
- [Average Transactions Per Purchaser](../references/metrics/avg_transactions_per_purchaser.md) — The average number of transactions made by purchasers.
- [Average Pageviews](../references/metrics/avg_pageviews.md) — The average number of pageviews per user.
- [Average Spend Per Purchase Session By User](../references/metrics/avg_spend_per_purchase_session_by_user.md) — The average amount of money spent per purchase session for each individual user.
- [Overall Average Spend Per Purchase Session](../references/metrics/overall_avg_spend_per_purchase_session.md) — The overall average amount spent across all unique purchase sessions.

# Schema
The `events_YYYYMMDD` table, created daily, and `events_intraday_YYYYMMDD` (for streaming export) contain the following fields:

## event
The event fields contain information that uniquely identifies an event.

- `batch_event_index` (INTEGER): A number indicating the sequential order of each event within a batch based on their order of occurrence on the device.
- `batch_ordering_id` (INTEGER): A monotonically increasing number that is incremented each time a network request is sent from a given page.
- `batch_page_id` (INTEGER): A sequential number assigned to a page that increases for each subsequent page within an engagement.
- `event_date` (STRING): The date when the event was logged (YYYYMMDD format in the registered timezone of your app).
- `event_timestamp` (INTEGER): The time (in microseconds, UTC) when the event was received by Google Analytics. Multiple events can share the same timestamp if sent in the same request.
- `event_previous_timestamp` (INTEGER): The time (in microseconds, UTC) when the previous event happened.
- `event_name` (STRING): The name of the event.
- `event_value_in_usd` (FLOAT): The currency-converted value (in USD) of the event's "value" parameter.
- `event_bundle_sequence_id` (INTEGER): The sequential ID of the bundle in which these events were uploaded.
- `event_server_timestamp_offset` (INTEGER): Timestamp offset between collection time and upload time in micros.

### event_params RECORD
The `event_params` RECORD can store campaign-level and contextual event parameters as well as any user-defined event parameters. The `event_params` RECORD is repeated for each key that is associated with an event.

- `event_params.key` (STRING): The name of the event parameter.
- `event_params.value` (RECORD): A record containing the event parameter's value.
    - `event_params.value.string_value` (STRING): If the event parameter is represented by a string, such as a URL or campaign name, it is populated in this field.
    - `event_params.value.int_value` (INTEGER): If the event parameter is represented by an integer, it is populated in this field.
    - `event_params.value.double_value` (FLOAT): If the event parameter is represented by a double value, it is populated in this field.
    - `event_params.value.float_value` (FLOAT): If the event parameter is represented by a floating point value, it is populated in this field. This field is not currently in use.
- `event_params.key = 'page_location'` (STRING): The URL of the page viewed.
- `event_params.key = 'page_title'` (STRING): The title of the page viewed.
- `event_params.key = 'ga_session_id'` (INTEGER): The ID of the Google Analytics session.
- `event_params.key = 'ga_session_number'` (INTEGER): The sequential number of the session for a user.

### item_params RECORD
The `item_params` RECORD can store item parameters as well as any user-defined item parameters.

- `item_params.key` (STRING): The name of the item parameter.
- `item_params.value` (RECORD): A record containing the item parameter’s value.
    - `item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.
    - `item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.
    - `item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.
    - `item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.

## user
The user fields contain information that uniquely identifies the user associated with the event.

- `is_active_user` (BOOLEAN): Whether the user was active (True) or inactive (False) at any point in the calendar day. This field is only populated in the daily tables (`events_YYYYMMDD`).
- `user_id` (STRING): The unique ID assigned to a user.
- `user_pseudo_id` (STRING): The pseudonymous id (e.g., app instance ID) for the user. A unique identifier that is assigned to a user when they first open the app or visit the site.
- `user_first_touch_timestamp` (INTEGER): The time (in microseconds) at which the user first opened the app or visited the site.

### privacy_info fields
The `privacy_info` fields contain information based on the consent status of a user when consent mode is enabled.

- `privacy_info.ads_storage` (STRING): Whether ad targeting is enabled for a user. Possible values: Yes, No, Unset
- `privacy_info.analytics_storage` (STRING): Whether Analytics storage is enabled for the user. Possible values: Yes, No, Unset
- `privacy_info.uses_transient_token` (STRING): Whether a web user has denied Analytics storage and the developer has enabled measurement without cookies based on transient tokens in server data. Possible values: Yes, No, Unset

### user_properties RECORD
The `user_properties` RECORD contains any user properties that you have set. It is repeated for each key that is associated with a user.

- `user_properties.key` (STRING): The name of the user property.
- `user_properties.value` (RECORD): A record for the user property value.
    - `user_properties.value.string_value` (STRING): The string value of the user property.
    - `user_properties.value.int_value` (INTEGER): The integer value of the user property.
    - `user_properties.value.double_value` (FLOAT): The double value of the user property.
    - `user_properties.value.float_value` (FLOAT): This field is currently unused.
    - `user_properties.value.set_timestamp_micros` (INTEGER): The time (in microseconds) at which the user property was last set.

### user_ltv RECORD
The `user_ltv` RECORD contains Lifetime Value information about the user. This RECORD is not populated in intraday tables.

- `user_ltv.revenue` (FLOAT): The Lifetime Value (revenue) of the user. This field is not populated in intraday tables.
- `user_ltv.currency` (STRING): The Lifetime Value (currency) of the user. This field is not populated in intraday tables.

## device
The device RECORD contains information about the device from which the event originated.

- `device.category` (STRING): The device category (mobile, tablet, desktop).
- `device.mobile_brand_name` (STRING): The device brand name.
- `device.mobile_model_name` (STRING): The device model name.
- `device.mobile_marketing_name` (STRING): The device marketing name.
- `device.mobile_os_hardware_model` (STRING): The device model information retrieved directly from the operating system.
- `device.operating_system` (STRING): The operating system of the device.
- `device.operating_system_version` (STRING): The OS version.
- `device.vendor_id` (STRING): IDFV (present only if IDFA is not collected).
- `device.advertising_id` (STRING): Advertising ID/IDFA.
- `device.language` (STRING): The OS language.
- `device.time_zone_offset_seconds` (INTEGER): The offset from GMT in seconds.
- `device.is_limited_ad_tracking` (BOOLEAN): The device's Limit Ad Tracking setting. On iOS14+, returns false if the IDFA is non-zero.
- `device.web_info.browser` (STRING): The browser in which the user viewed content.
- `device.web_info.browser_version` (STRING): The version of the browser in which the user viewed content.
- `device.web_info.hostname` (STRING): The hostname associated with the logged event.

## geo
The geo RECORD contains information about the geographic location where the event was initiated.

- `geo.continent` (STRING): The continent from which events were reported, based on IP address.
- `geo.sub_continent` (STRING): The subcontinent from which events were reported, based on IP address.
- `geo.country` (STRING): The country from which events were reported, based on IP address.
- `geo.region` (STRING): The region from which events were reported, based on IP address.
- `geo.metro` (STRING): The metro from which events were reported, based on IP address.
- `geo.city` (STRING): The city from which events were reported, based on IP address.

## app_info
The `app_info` RECORD contains information about the app in which the event was initiated.

- `app_info.id` (STRING): The package name or bundle ID of the app.
- `app_info.firebase_app_id` (STRING): The Firebase App ID associated with the app.
- `app_info.install_source` (STRING): The store that installed the app.
- `app_info.version` (STRING): The app's versionName (Android) or short bundle version.

## collected_traffic_source
The `collected_traffic_source` RECORD contains the traffic source data that was present within the events that were collected.

- `manual_campaign_id` (STRING): The manual campaign id (utm_id) that was collected with the event.
- `manual_campaign_name` (STRING): The manual campaign name (utm_campaign) that was collected with the event.
- `manual_source` (STRING): The manual campaign source (utm_source) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.
- `manual_medium` (STRING): The manual campaign medium (utm_medium) that was collected with the event. Also includes parsed parameters from referral params, not just UTM values.
- `manual_term` (STRING): The manual campaign keyword/term (utm_term) that was collected with the event.
- `manual_content` (STRING): The additional manual campaign metadata (utm_content) that was collected with the event.
- `manual_creative_format` (STRING): The manual campaign creative format (utm_creative_format) that was collected with the event.
- `manual_marketing_tactic` (STRING): The manual campaign marketing tactic (utm_marketing_tactic) that was collected with the event.
- `manual_source_platform` (STRING): The manual campaign source platform (utm_source_platform) that was collected with the event.
- `gclid` (STRING): The Google click identifier that was collected with the event.
- `dclid` (STRING): The DoubleClick Click Identifier for Display and Video 360 and Campaign Manager 360 that was collected with the event.
- `srsltid` (STRING): The Google Merchant Center identifier that was collected with the event.

## session_traffic_source_last_click
The `session_traffic_source_last_click` RECORD contains the last-click attributed session traffic source data across Google ads and manual contexts, where available.

- `session_traffic_source_last_click.manual_campaign.campaign_id` (STRING): The ID of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.campaign_name` (STRING): The name of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.medium` (STRING): The medium of the last clicked manual campaign (e.g., paid search, organic search, email).
- `session_traffic_source_last_click.manual_campaign.term` (STRING): The keyword/search term of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.content` (STRING): Additional metadata of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.source_platform` (STRING): The platform of the last clicked manual campaign (e.g., search engine, social media).
- `session_traffic_source_last_click.manual_campaign.source` (STRING): The specific source within the platform of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.creative_format` (STRING): The format of the creative of the last clicked manual campaign.
- `session_traffic_source_last_click.manual_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked manual campaign.
- `session_traffic_source_last_click.google_ads_campaign.customer_id` (STRING): The customer ID associated with the Google Ads account.
- `session_traffic_source_last_click.google_ads_campaign.account_name` (STRING): The name of the Google Ads account.
- `session_traffic_source_last_click.google_ads_campaign.campaign_id` (STRING): The ID of the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.campaign_name` (STRING): The name of the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.ad_group_id` (STRING): The ID of the ad group within the Google Ads campaign.
- `session_traffic_source_last_click.google_ads_campaign.ad_group_name` (STRING): The name of the ad group within the Google Ads campaign.
- `session_traffic_source_last_click.cross_channel_campaign.campaign_name` (STRING): The name of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.campaign_id` (STRING): The ID of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.source_platform` (STRING): The platform of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.source` (STRING): The specific source within the platform of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.cross_channel_campaign.medium` (STRING): The medium of the last clicked cross-channel campaign.
- `session_traffic_source_last_click.sa360_campaign.campaign_name` (STRING): The name of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.source` (STRING): The specific source within the platform of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.medium` (STRING): The medium of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.ad_group_id` (STRING): The ID of the ad group within the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.ad_group_name` (STRING): The name of the ad group within the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.campaign_id` (STRING): The ID of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.creative_format` (STRING): The format of the creative of the last clicked SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.engine_account_name` (STRING): The name of the SA360 engine account.
- `session_traffic_source_last_click.sa360_campaign.engine_account_type` (STRING): The type of engine account containing the SA360 campaign.
- `session_traffic_source_last_click.sa360_campaign.manager_account_name` (STRING): The name of the SA360 manager account.
- `session_traffic_source_last_click.dv360_campaign.advertiser_id` (STRING): The ID of the DV360 advertiser.
- `session_traffic_source_last_click.dv360_campaign.advertiser_name` (STRING): The name of the DV360 advertiser.
- `session_traffic_source_last_click.dv360_campaign.campaign_id` (STRING): The ID of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.campaign_name` (STRING): The name of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_id` (STRING): The ID of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_format` (STRING): The format of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.creative_name` (STRING): The name of the creative of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.marketing_tactic` (STRING): The marketing tactic of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.exchange_id` (STRING): The exchange ID of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.exchange_name` (STRING): The exchange name of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.insertion_order_id` (STRING): The ID of the insertion order in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.insertion_order_name` (STRING): The name of the insertion order in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.line_item_id` (STRING): The ID of the line item in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.line_item_name` (STRING): The name of the line item in the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.partner_id` (STRING): The ID of the DV360 partner.
- `session_traffic_source_last_click.dv360_campaign.partner_name` (STRING): The name of the DV360 partner.
- `session_traffic_source_last_click.dv360_campaign.source` (STRING): The specific source within the platform of the last clicked DV360 campaign.
- `session_traffic_source_last_click.dv360_campaign.medium` (STRING): The medium of the last clicked DV360 campaign.
- `session_traffic_source_last_click.cm360_campaign.account_id` (STRING): The ID of the CM360 account.
- `session_traffic_source_last_click.cm360_campaign.account_name` (STRING): The name of the CM360 account.
- `session_traffic_source_last_click.cm360_campaign.advertiser_id` (STRING): The ID of the CM360 advertiser.
- `session_traffic_source_last_click.cm360_campaign.advertiser_name` (STRING): The name of the CM360 advertiser.
- `session_traffic_source_last_click.cm360_campaign.campaign_id` (STRING): The ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.campaign_name` (STRING): The name of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_id` (STRING): The ID of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_format` (STRING): The format of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_name` (STRING): The name of the creative of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_type` (STRING): The creative type of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_type_id` (STRING): The creative type ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.creative_version` (STRING): The creative version of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_id` (STRING): The ID of the placement of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_cost_structure` (STRING): The placement cost structure of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.placement_name` (STRING): The name of the placement of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.rendering_id` (STRING): The rendering ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.site_id` (STRING): The site ID of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.site_name` (STRING): The site name of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.source` (STRING): The specific source of the last clicked CM360 campaign.
- `session_traffic_source_last_click.cm360_campaign.medium` (STRING): The medium of the last clicked CM360 campaign.

## traffic_source
The `traffic_source` RECORD contains information about the traffic source that first acquired the user. This record is not populated in intraday tables.

- `traffic_source.name` (STRING): Name of the marketing campaign that first acquired the user. This field is not populated in intraday tables.
- `traffic_source.medium` (STRING): Name of the medium (paid search, organic search, email, etc.) that first acquired the user. This field is not populated in intraday tables.
- `traffic_source.source` (STRING): Name of the network that first acquired the user. This field is not populated in intraday tables.

## stream and platform
The stream and platform fields contain information about the stream and the app platform.

- `stream_id` (STRING): The numeric ID of the data stream from which the event originated.
- `platform` (STRING): The data stream platform (Web, IOS or Android) from which the event originated.

## ecommerce
This ecommerce RECORD contains information about any ecommerce events that have been setup on a website or app.

- `ecommerce.total_item_quantity` (INTEGER): Total number of items in this event, which is the sum of items.quantity.
- `ecommerce.purchase_revenue_in_usd` (FLOAT): Purchase revenue of this event, represented in USD with standard unit. Populated for purchase event only.
- `ecommerce.purchase_revenue` (FLOAT): Purchase revenue of this event, represented in local currency with standard unit. Populated for purchase event only.
- `ecommerce.refund_value_in_usd` (FLOAT): The amount of refund in this event, represented in USD with standard unit. Populated for refund event only.
- `ecommerce.refund_value` (FLOAT): The amount of refund in this event, represented in local currency with standard unit. Populated for refund event only.
- `ecommerce.shipping_value_in_usd` (FLOAT): The shipping cost in this event, represented in USD with standard unit.
- `ecommerce.shipping_value` (FLOAT): The shipping cost in this event, represented in local currency.
- `ecommerce.tax_value_in_usd` (FLOAT): The tax value in this event, represented in USD with standard unit.
- `ecommerce.tax_value` (FLOAT): The tax value in this event, represented in local currency with standard unit.
- `ecommerce.transaction_id` (STRING): The transaction ID of the ecommerce transaction.
- `ecommerce.unique_items` (INTEGER): The number of unique items in this event, based on item_id, item_name, and item_brand.

## items
The `items` RECORD contains information about items included in an event. It is repeated for each item.

- `items.item_id` (STRING): The ID of the item.
- `items.item_name` (STRING): The name of the item.
- `items.item_brand` (STRING): The brand of the item.
- `items.item_variant` (STRING): The variant of the item.
- `items.item_category` (STRING): The category of the item.
- `items.item_category2` (STRING): The sub category of the item.
- `items.item_category3` (STRING): The sub category of the item.
- `items.item_category4` (STRING): The sub category of the item.
- `items.item_category5` (STRING): The sub category of the item.
- `items.price_in_usd` (FLOAT): The price of the item, in USD with standard unit.
- `items.price` (FLOAT): The price of the item in local currency.
- `items.quantity` (INTEGER): The quantity of the item. Quantity set to 1 if not specified.
- `items.item_revenue_in_usd` (FLOAT): The revenue of this item, calculated as price_in_usd * quantity. It is populated for purchase events only, in USD with standard unit.
- `items.item_revenue` (FLOAT): The revenue of this item, calculated as price * quantity. It is populated for purchase events only, in local currency with standard unit.
- `items.item_refund_in_usd` (FLOAT): The refund value of this item, calculated as price_in_usd * quantity. It is populated for refund events only, in USD with standard unit.
- `items.item_refund` (FLOAT): The refund value of this item, calculated as price * quantity. It is populated for refund events only, in local currency with standard unit.
- `items.coupon` (STRING): Coupon code applied to this item.
- `items.affiliation` (STRING): A product affiliation to designate a supplying company or brick and mortar store location.
- `items.location_id` (STRING): The location associated with the item.
- `items.item_list_id` (STRING): The ID of the list in which the item was presented to the user.
- `items.item_list_name` (STRING): The name of the list in which the item was presented to the user.
- `items.item_list_index` (STRING): The position of the item in a list.
- `items.promotion_id` (STRING): The ID of a product promotion.
- `items.promotion_name` (STRING): The name of a product promotion.
- `items.creative_name` (STRING): The name of a creative used in a promotional spot.
- `items.creative_slot` (STRING): The name of a creative slot.

### item_params RECORD (nested under items)
The `item_params` RECORD stores the custom item parameters that you defined.

- `items.item_params.key` (STRING): The name of the item parameter.
- `items.item_params.value` (RECORD): A record containing the item parameter’s value.
    - `items.item_params.value.string_value` (STRING): If the item parameter is represented by a string, it is populated in this field.
    - `items.item_params.value.int_value` (INTEGER): If the item parameter is represented by an integer, it is populated in this field.
    - `items.item_params.value.double_value` (FLOAT): If the item parameter is represented by a double value, it is populated in this field.
    - `items.item_params.value.float_value` (FLOAT): If the item parameter is represented by a floating point value, it is populated in this field.

## publisher (Early access only)
The publisher RECORD contains information about events sourced from a publisher integration related to the display of ads, that is, AdMob.

- `publisher` (RECORD): A record of publisher data coming from AdMob. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_revenue_in_usd` (FLOAT): Estimated ad revenue resulting from this event, represented in USD. Populated for ad impression events only. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_format` (STRING): Describes the way ads appeared and where they were located. Typical formats include ‘Interstitial’, ‘Banner’, ‘Rewarded’, and ‘Native advanced’. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_source_name` (STRING): The source network that served an ad. Typical sources include, ‘AdMob Network’, ‘Meta audience Network’, and ‘Mediated house ads’. This field is not populated in intraday tables and Fresh Daily BigQuery export.
- `publisher.ad_unit_id` (STRING): The name you chose to describe this Ad unit. Ad units are containers that you place in your apps to show ads to users. This field is not populated in intraday tables and Fresh Daily BigQuery export.

# Joins
- [Google Analytics Events to Google Ads Clicks](../references/joins/events___ads_clickstats.md) — join on `collected_traffic_source.gclid` to attach Google Ads data to events.

# Citations
- https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset
- https://support.google.com/analytics/answer/7029846
- https://developers.google.com/analytics/bigquery/basic-queries
- https://developers.google.com/analytics/bigquery/advanced-queries


================================================
FILE: toolbox\mdcode\demo\okf\catalog\tables\index.md
================================================
# BigQuery Table

* [Events table (Google Analytics BigQuery Export)](events_.md) - Contains Google Analytics event export data from the `ga4_obfuscated_sample_ecommerce` dataset.


================================================
FILE: toolbox\mdcode\demo\okf\cleanup.ts
================================================
import * as cp from 'child_process';
import * as kcmd from 'kcmd';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const location = context.location;
const entryGroup = 'okf_ga4';

function dataplex(cmd: string, data: string|null=null) {
  cmd = 'gcloud -q dataplex ' + cmd + ` --project ${project} --location ${location}`;
  cp.execSync(cmd, { encoding: 'utf8', input: data ?? undefined, stdio: 'inherit'});
}

dataplex(`entry-groups delete ${entryGroup}`);
console.log(`Deleted entry group ${entryGroup}`);


================================================
FILE: toolbox\mdcode\demo\okf\setup.ts
================================================
import * as cp from 'child_process';
import * as path from 'node:path';
import * as kcmd from 'kcmd';
import { YAML } from 'bun';

const context = kcmd.gcp.ApiContext.default();
const project = context.project;
const location = context.location;
const entryGroup = 'okf_ga4';

function dataplex(cmd: string, data: string|null=null) {
  cmd = 'gcloud dataplex ' + cmd + ` --project ${project} --location ${location}`;
  cp.execSync(cmd, { encoding: 'utf8', input: data ?? undefined, stdio: 'inherit'});
}

try {
  dataplex(`entry-groups create ${entryGroup}`);
  console.log(`Created empty entry group ${entryGroup}`);
  console.log();
}
catch {
  // Might already exist
}

await Bun.file(path.join(process.cwd(), 'catalog.yaml')).write(YAML.stringify({
  scope: `kb.${project}.${location}.${entryGroup}`,
  snapshot: {
    entries: [
      'dataplex-types.global.generic'
    ],
    aspects: [
      'dataplex-types.global.generic',
      'dataplex-types.global.overview'
    ]
  },
  publishing: {
    entries: [
      'dataplex-types.global.generic'
    ],
    aspects: [
      'dataplex-types.global.generic',
      'dataplex-types.global.overview'
    ]
  }
}, null, 2));
console.log('Created catalog.yaml manifest');


================================================
FILE: toolbox\mdcode\demo\README.md
================================================
# Demo

## Configuration

You will need a cloud project with permissions to use BigQuery and Knowledge Catalog APIs.

```bash
export DEMO_CLOUD_PROJECT=<GCP_PROJECT_ID>
```

Ensure that gcloud is installed and configured.

```bash
gcloud auth application-default login
gcloud config set compute/region us-central1
gcloud config set project $DEMO_CLOUD_PROJECT
```

## BigQuery Dataset

This demo demonstrates working with metadata for BigQuery resources (dataset and table).

**Setup**

* Creates a BigQuery dataset (`demo_ecommerce`) and a table (`events`) based on BigQuery
  sample data in your cloud project.
* Creates a `catalog.yaml` manifest to specify the local catalog snapshot.

```bash
bun setup.ts
cat catalog.yaml
```

**Create Metadata Snapshot**

* Pull metadata from Knowledge Catalog

```bash
../../dist/kcmd pull
ls -R catalog
cat catalog/$DEMO_CLOUD_PROJECT.demo_ecommerce.yaml
```

**Modify Metadata Snapshot**

* Either manually edit the file, or use the following command which adds a dummy
  overview aspect.

```bash
bun update.ts catalog/$DEMO_CLOUD_PROJECT.demo_ecommerce.yaml
cat catalog/$DEMO_CLOUD_PROJECT.demo_ecommerce.yaml
```

**Publish Metadata Snapshot**

* Push metadata to Knowledge Catalog

```bash
../../dist/kcmd push
```

**Cleanup**

* Deletes the BigQuery resources created for the demo

```bash
bun cleanup.ts
```

## Knowledge Base

This demo demonstrates working with a Knowledge Base managed in Knowledge Catalog.

**Setup**

* Creates a Dataplex EntryGroup (`demo_kb`) and a set of document entries within it.
* Creates a `catalog.yaml` manifest to specify the local catalog snapshot.

```bash
bun setup.ts
cat catalog.yaml
```

**Create Metadata Snapshot**

* Pull metadata from Knowledge Catalog

```bash
../../dist/kcmd pull
ls -R catalog
cat catalog/index.md
```

**Modify Metadata Snapshot**

* Either manually edit the file, or use the following command which creates a placeholder
  demo update to the content of the `index` entry using the `kcmd` (metadata as code)
  library.

```bash
bun update.ts
cat catalog/index.md
```

**Publish Metadata Snapshot**

* Push metadata to Knowledge Catalog

```bash
../../dist/kcmd push
```

**Cleanup**

* Deletes the Dataplex EntryGroup

```bash
bun cleanup.ts
```

## OKF Wiki

This demo demonstrates publishing an [Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/tree/main/okf)
wiki bundle (a directory of markdown files with YAML frontmatter) into a
Knowledge Catalog EntryGroup via the Documents Layout. The bundle in
`okf/catalog/` is a GA4 sample with indexes, references, a dataset, and a
table, 17 markdown files in total. The Documents Layout maps each `.md`
file to an entry whose name is derived from the file path, with the
markdown body stored on the `dataplex-types.global.overview` aspect.

**Setup**

* Creates an empty Dataplex EntryGroup (`okf_ga4`).
* Creates a `catalog.yaml` manifest pointing at the EntryGroup.
* The `catalog/` directory is already populated with the GA4 markdown bundle.

```bash
bun setup.ts
cat catalog.yaml
ls -R catalog
```

**Publish Metadata Snapshot**

* Push the bundled markdown to Knowledge Catalog. Entry names mirror the
  file path (e.g. `references/metrics/event_count.md` &rarr; entry
  `references/metrics/event_count`). Custom `type:` values in frontmatter
  that aren't valid Dataplex type refs fall back to
  `dataplex-types.global.generic`.

```bash
../../dist/kcmd push
```

**Modify Metadata Snapshot**

* Edit any markdown file under `catalog/` directly. Both frontmatter
  fields (`title`, `description`, `tags`) and the markdown body can be
  changed, then push again.

```bash
../../dist/kcmd push
```

**Cleanup**

* Deletes the Dataplex EntryGroup

```bash
bun cleanup.ts
```


================================================
FILE: toolbox\mdcode\docs\concept.md
================================================
# Metadata as Code

## Overview

This document details Metadata as Code, a core Knowledge Catalog capability
designed to provide agent builders, data stewards and data producers, a source
code artifact-based UX for metadata management and context engineering. Metadata
as Code enables users and agents to author, manage, and consume metadata
artifacts using developer-friendly practices such as versioning and CI/CD as
part of their developer repository, and activate context enrichment agents on a
standardized metadata format.

This approach described in this document builds on top of broadly supported
markdown and YAML, and simple file organization structure that enables
bi-directional sync capability with Knowledge Catalog service by layering on top
of existing Import/Export capabilities, and factors in extensibility that is a
core tenet of the Catalog platform.

**High-level Plan**
We will pick a YAML representation that largely aligns with the Entry/EntryLink
+ Aspect data model in Knowledge Catalog to represent both 1st party and 3rd
party metadata. Unstructured text content in aspects will be represented as
sidecar markdown files. Local files are organized in a directory structure
aligned with the resource hierarchy of assets represented by the metadata. Users
can provide configurations to sync (download/snapshot and publish/deploy)
metadata from and to the service.

### Tenets

* **Optimized for consumption by users and agents alike.**
  Metadata as code artifacts should be easy to read and modify as well as
  naturally fit as files within a developer's repository and support use-cases
  such as versioning, diffing, CI/CD etc. In support of ease of consumption,
  users should be able to organize their metadata in logical and modular slices,
  while the layout of metadata facilitates organization and navigation.

* **Optimized for tools ecosystem**
  The code artifacts should benefit from existing IDE and tool capabilities such
  as previews, navigation, etc.

* **Enable bi-directional sync with Knowledge Catalog and metadata fidelity**
  The code artifacts should be amenable to serving as authoring and management
  source of truth. They should enable a user to fully inspect and author
  metadata that exists in the Catalog service. Metadata as code artifacts should
  be able to represent 1st and 3rd party entries, aspects and the links between
  them.

* **Embeddable and Consistent Representation**
  Users may operate on the metadata in a standalone manner in their workspace
  for example, in the context of an enrichment workflow. Alternatively, the
  metadata may be part of a larger project, such as a Dataform/DBT pipeline,
  a BigQuery semantic graph etc. We should strive for a format and
  representation that can largely be consistent across these use-cases, and be
  used independently or embedded.

### Goals

**Source artifact for agents and human-in-the-loop review workflows**
We will offer an agentic capability for metadata curation and enrichment. The
agent will operate over the code representation of metadata to read it, and
update it. Users can also track changes, review it, and if needed, directly
modify it in an interactive loop.

Metadata as Code is a pre-requisite capability to allow such agents to be built
and integrated into a standardized workflow and UX.

**Source artifact for metadata sidecars**
Similar to the above, we are also collectively building agents to streamline
building Semantic Graphs or LookML models, and Data Pipelines. These efforts are
building their use-case specific code artifacts and agent flows.

Metadata as code artifacts should naturally live within a larger project. This
allows users to work with a standard representation. It allows the enrichment
agents to be plugged into the larger agent flow as a sub-agent or agent tool.

**Context versioning**
Context engineering is central to agent building. Agent builders will need to be
able to operate on their context as source code, evolve it in a versioned
manner, and deploy it as versioned resources.

Metadata as code provides the source code UX and workflows (by virtue of being
hostable within a source control repository and using the existing versioning
capabilities). Developers can then deploy the context to co-existing deployed
versions to be able to run a/b tests and evals, and have agents pick the
appropriate version.

### Non-Goals

**Hierarchical Context Serving Store or Index**
The metadata as code model described in this document is aligned with the data
model of metadata within Knowledge Catalog that organizes metadata to mirror
resources in the integrated source systems. Specifically this means metadata
organized as Entries and EntryLinks with aspects, within an EntryGroup (both
system and user-managed) in a specific project.

Context serving or searching use-cases would benefit from a different
organization of metadata (eg. arbitrary hierarchy of content organized by data
stores, use-cases, etc.). Conceivably, we might find such hierarchies naturally
present in Document repositories such as Drive, or g3doc/wikis, or process
metadata in Knowledge Catalog to produce a serving-ready version of metadata.

## Use-cases

This is a high-level mention of some of the use-cases where Metadata-as-Code
should help with context engineering for agents.

**Knowledge Catalog Enrichment Agent**
Knowledge Catalog will offer an off-the-shelf enrichment agent that is pluggable
with user-provided instructions, skills and tools. This will allow users to plug
in their organizational data sources, including code repositories, queries,
documentation in wikis, drive etc., internal Q&A channels etc.

The enrichment agent operates on code artifacts, providing a mechanism for users
to iterate on enrichment, as well as apply human review steps, and integration
with git-based version control and CI/CD mechanisms for deployment to make the
updates live.

**Semantic Graph Business/Semantic Enrichment**
We are introducing Semantic Graphs as the mechanism for users to build semantic
models - defining entities and their relationships - that will be consumed by CA
agents to ground their behavior and resulting queries in the organizational
truth.

The graph will allow users to define entities as nodes and relationships as
edges along with additional properties, including computed values and measures.
Users will want to attach additional meaning such as glossary term definitions
as part of their graph authoring UX. Metadata-as-Code will provide a mechanism
for users to co-locate their graph definition and catalog augmentations, and
deploy them collectively.

**Metadata propagation and curation in Data Pipelines**
We are building a Data Engineering agent to streamline the creation and
productionization of data pipelines. Data engineers should be able to leverage
as well as publish metadata along with the deployment of their pipelines and
data sources and sinks. This will ensure the resulting data is more ready for
agentic consumption and allow for co-locating and unifying the UX for the
transforms and the metadata flow within a pipeline project.

**Database Context Set**
We are introducing ContextSet and related functionality (Query Templates,
Examples, etc.) to enable database admins and developers to engineer context for
their NLD and NLA agents.

This discussion is early. The observation is Metadata-as-Code offers a potential
mechanism to unify the UX for context authoring, while supporting different
service models (EntryGroup and ContextSetGroup) for purposes of storage, search
and serving.

## File Organization

Metadata is organized within a directory. The directory represents a local
snapshot of a subset of metadata in Catalog. It is the unit of synchronization
with the metadata within the service.

This directory contains a manifest file, named catalog.yaml, that provides
structured configuration that can direct tools on handling the individual
artifacts contained within the directory. It contains sub-directories that
contain individual entries and entry links.

Depending on the **scope** (see manifest below), files are organized in one of
two layout formats (and potentially extendable to other formats in future):

**Standard Layout:**
Used for `bq-dataset` and `entryGroup` scopes. This is a hybrid disk
organization where the metadata structure is stored in a main YAML file per
entry, and unstructured aspects (like Overview) are placed in sidecar Markdown
files.

```
path/to/root/
├── catalog.yaml                      # Catalog metadata, processing config/directives
└── catalog/                          # Contains all the Entries and EntryLinks
    └── <dir1>/<dir2>
        ├── <entry-id1>.yaml          # Single-file entry with all metadata contained within
        ├── <entry-id2>.yaml          # Multi-file entry with unstructured metadata split into
        └── <entry-id2>.<aspect>.md   # markdown sidecar files for unstructured aspects
```

**Documents Layout:**
Used for `kb` scopes. This is a Markdown-first disk organization where the
main entry is represented as a single Markdown file, with metadata structured in
the YAML frontmatter and the main overview aspect promoted to the Markdown body.

```
path/to/root/
├── catalog.yaml                      # Catalog metadata, processing config/directives
└── catalog/                          # Contains all the Entries and EntryLinks
    └── <dir1>/<dir2>
        └── <entry-id1>.md            # Single markdown file: structured metadata in
                                      # frontmatter, Overview in body
```

**NOTE: IDE Workspace**
An IDE workspace, like the one in Antigravity, can include one or more such
directories, organized according to the user's preferences.

### Manifest - Catalog.yaml

The manifest file captures all configuration related information. This includes:

* The resource that the metadata snapshot corresponds to (the `scope`). This
  resource can either be a resource identified in a source system such as BigQuery
  (e.g. a Dataset, or a set of Tables, whose metadata is managed in Catalog in a
  system EntryGroup, such as @bigquery), a user-managed EntryGroup containing
  user created entries, or a user-managed EntryGroup representing a knowledge base containing markdown-based documentation entries.

* The physical layout of the catalog snapshot on disk is automatically determined
  by the type of the scope:
  * A `bq-dataset` or `entryGroup` scope uses the standard layout.
  * A `kb` scope uses the wiki layout.

* A set of type aliases to make it easy to refer to various catalog constructs
  such as entry-types, aspect-types, link-types, glossaries, glossary-terms, etc.

* The type of entries, aspects, and links to include into the local snapshot
  when downloading metadata from the service.

* The subset of entries, aspects, and links to publish back to the service.

**catalog.yaml**
```
scope: <type>.<name>              # The resources in the snapshot. Examples:
                                  # scope: entryGroup.<projectId>.<locationId>.<entryGroupId>
                                  # scope: bq-dataset.<projectId>.<datasetId>
                                  # scope: kb.<projectId>.<locationId>.<entryGroupId>
                                  # Or multiple BigQuery datasets in array list format:
                                  # scope:
                                  #   - bq-dataset.<projectId>.<datasetId1>
                                  #   - bq-dataset.<projectId>.<datasetId2>


aliases:                          # Optional. Can always use 3-part fully qualified references.
                                  # NOTE: All built-in types have predefined simple aliases.
  ca-guidelines:
    aspect: data-agents-project.global.ca-guidelines
  ecommerce:
    glossary: data-gov-project.global.ecommerce-glossary

# Defines the specific metadata to be retrieved locally from Knowledge Catalog.
# NOTE: Required aspects of listed entry types are implicitly included.
snapshot:
  entries:
  - bigquery-dataset
  - bigquery-table
  aspects:
  - overview
  - descriptions
  - queries
  - ca-guidelines
  entryLinks:
  - definition

# Optional configuration to identify which types should be published.
# This must be a subset of the types specified in the snapshot configuration.
publishing:
  entries:
  aspects:
  - ca-guidelines
  entryLinks:
  - definition
```

**NOTE: Type References**
Metadata content has several references to either the meta-model resources
(entry, aspect, and link types) or other catalog constructs (such as
glossaries). Syntax is provided to streamline these references.

* References take the form of `<projectId>.<locationId>.<resourceId>`. There is
  no need for referring to the verbose collection names themselves
  (projects/<projectId>/locations/<locationId>/<collection>/<resourceId>)
* Aliases can be defined to create single identifier names that are resolved
  when metadata is processed.
  * These are optional. Users can choose to use the fully-qualified references
    if that is the preference.
  * All built-in Dataplex types have pre-defined aliases without requiring the
    user to declare them. Eg. bigquery-table resolves to
    dataplex-types.global.bigquery-table.
* Use project IDs, not project numbers when generating metadata.

### Entry File Names

The name of the file is used to identify the Entry uniquely. The following rules
are followed to derive file names from EntryIds.

**Ingested Entries** use the service-qualified full resource name as EntryId.
Inferrable segments (Service name and Location from EntryGroup) and Collection
names (where possible) will be stripped out.

| bigquery.googleapis.com/ projects/projectId/ datasets/datasetId/ tables/tableId | projectId.datasetId/tableId.yaml | Aligns with physical hierarchy. Common case for context use-cases. Avoid a type segment in directory path to optimize for most common resource when a collection has multiple types of sub-resources |
| bigquery.googleapis.com/ projects/projectId/ datasets/datasetId/ routines/routineId | routines/projectId.datasetId/routineId.yaml | Moved into sub-directory to avoid potential conflicts between table and routine names |
| cloudsql.googleapis.com/ projects/projectId/ locations/locationId/ instances/instanceId/ databases/databaseId | instanceId/databaseId/tableId.yaml | Example included to demonstrate how EntryIds and corresponding file names treat control plane and dataplane ids together |


**3rd party Custom Entries** may use a similar REST API name of the resource
(recommended when applicable) or arbitrary path-identifier (more likely
scenario); We align the metadata file path to the Entry Id as there are no
conventions to apply here.

| Entry Id | File Path | NOTES |
| :---- | :---- | :---- |
| hive.walmart.com/ services/serviceId/ databases/databaseId/ tables/tableId | hive.walmart.com/ services/serviceId/ databases/databaseId/ tables/tableId.yaml |  |
| name-part1/name-part2 | name-part1/name-part2.yaml | Likely the common pattern for custom context overlays and bundles. We ensure that it is clean in directory layout. |
| simple-name | simple-name.yaml |  |

### File Extensions

* **Standard Layout (for `bq-dataset` and `entryGroup` scopes)**: Entry files
use the `.yaml` extension (e.g., `tableId.yaml`), with unstructured aspects
optionally stored in sidecar `.md` files (e.g., `tableId.overview.md`).

* **Document Layout (for `kb` scopes)**: Entry files use the `.md` extension
(e.g., `tableId.md`), where the main YAML entry is replaced with a single
markdown file containing YAML frontmatter and the promoted unstructured aspect
(`Overview.content`) as its body.

### Entry File Layouts

Metadata files are primarily structured representations of entry catalog info
and aspects. Depending on the scope, the organization of these files differs:

* **Standard Layout (YAML + Markdown sidecars)**: Used for `bq-dataset` and
`entryGroup` scopes. Every entry is represented by at least a single YAML file
(`<entry-id>.yaml`) that captures entry metadata, information about associated
resource, and all aspects and links. Aspects containing unstructured rich-text
fields (like `overview`) are split into sidecar markdown files
(e.g., `<entry-id>.overview.md`).

* **Document Layout (Markdown Only)**: Used for `kb` scopes. The main YAML file is
replaced with a single markdown file (`<entry-id>.md`). The structured YAML
metadata moves entirely to the YAML frontmatter of this markdown file, and the
unstructured text of the `overview.content` aspect becomes the main markdown
body of the file.

#### Standard Layout

**Entry Resource Info + Entry Source + Aspects (`entry.yaml`)**
```yaml
id: <id>                                # Entry metadata
type: <entryType>
resource:                               # Entry.EntrySource
  name: <entrySource.name>
  displayName: <entrySource.displayName>
  description: <entrySource.description>
  labels:
    key: value
  location: <entrySource.location>
  parent: <entry.parent>
  ancestors: <entrySource.ancestors>
  createTime: <entrySource.createTime>
  updateTime: <entrySource.updateTime>

schema:
  fields:
  - name1: <schemaField.name>
    dataType: <schemaField.dataType>
    mode: <schemaField.mode>
    …
    links:                              # EntryLinks associated with Schema.path are inlined
      definition:                       # into a Schema field to leverage context of field
      - target: glossary.term           # specification

<aspect-type>:                          # In the general-case, each top-level field is an
  [aspect.data]                         # aspect. Nested field represents aspect.data

links:                                  # EntryLinks with this entry as source listed here
  <entryLink-type>:
  - target: <target-entry-reference>
    <aspect-type>:
      [aspect.data]
```

**Sidecar File per Aspect with Unstructured Text (`entry.overview.md`)**
```markdown
---
field: value                            # YAML frontmatter used to capture structured metadata
---
[aspect.data.rich-text-field]
```

#### Wiki Layout

In the wiki layout, the YAML content is placed in the YAML frontmatter
of a `.md` file, and the unstructured overview text (`overview.content`) is
promoted to the markdown body.

**Entry with Inlined Unstructured Text (`entry.md`)**
```markdown
---
id: <id>                                # Entry metadata in frontmatter
type: <entryType>
resource:                               # Entry.EntrySource
  name: <entrySource.name>
  displayName: <entrySource.displayName>
  description: <entrySource.description>
  labels:
    key: value
  location: <entrySource.location>
  parent: <entry.parent>
  ancestors: <entrySource.ancestors>
  createTime: <entrySource.createTime>
  updateTime: <entrySource.updateTime>

schema:
  fields:
  - name1: <schemaField.name>
    dataType: <schemaField.dataType>
    mode: <schemaField.mode>
    …
    links:
      definition:
      - target: glossary.term

<aspect-type>:                          # Other structured aspects
  [aspect.data]

overview:                               # Unstructured aspect's metadata
  userManaged: true
  links:
  - title: <title>
    url: <url>

links:                                  # EntryLinks with this entry as source
  <entryLink-type>:
  - target: <target-entry-reference>
    <aspect-type>:
      [aspect.data]
---
# <displayName or entry-id>

This is the main markdown body representing `overview.content` aspect data.
```

## Examples

### Enrichment Agent

This is an example for a BigQuery dataset and a table. For purposes of
illustration, the user is interested in working the metadata that is used in
Conversational Analytics, to enrich Overview documentation, Generated
descriptions, Suggested Queries, Agent Guidelines and Glossary associations, and
publish them back to Knowledge Catalog.

**workspace/catalog.yaml**
```
scope: bq-dataset.ecommerce-prod.ecommerce-dataset

resourceAliases:
  guidelines:
    aspect: data-agents-project.global.guidelines
  ecommerce:
    glossary: data-gov-project.global.ecommerce-glossary

snapshot:
  entries:
  - bigquery-dataset
  - bigquery-table
  aspects:
  - overview
  - descriptions
  - queries
  - guidelines
  - data-profile            # Included as local context, but not published
  - query-usage             # Same as above
  entryLinks:
  - definition

publishing:
  aspects:
  - overview
  - descriptions
  - queries
  - guidelines
  entryLinks:
  - definition
```

**workspace/catalog/ecommerce-prod.ecommerce-dataset.yaml**
```
id: bigquery.googleapis.com/projects/ecommerce-prod/datasets/ecommerce-dataset
type: bigquery-dataset
createTime: <createTime>
updateTime: <updateTime>

resource:
  name: projects/ecommerce-prod/datasets/ecommerce-dataset
  displayName: E-commerce Production Dataset
  description: ...
  labels:
    env: prod
  createTime: <entrySource.createTime>
  updateTime: <entrySource.updateTime>

bigquery-dataset:
  ...

descriptions:
  description: <generated dataset description>
```

**workspace/catalog/ecommerce-prod.ecommerce-dataset/orders.yaml**
```
id: bigquery.googleapis.com/projects/ecommerce-prod/datasets/ecommerce-dataset/tables/orders
type: bigquery-table
createTime: <createTime>
updateTime: <updateTime>

resource:
  name: projects/ecommerce-prod/datasets/ecommerce-dataset/tables/orders
  displayName: E-commerce Orders
  description: ...
  labels:
    env: prod
  createTime: <entrySource.createTime>
  updateTime: <entrySource.updateTime>

bigquery-table:
  ...
storage:
  ...

schema:
  fields:
  - name: id
    dataType: string
    mode: required
    links:
      definition:
      - target: ecommerce.order-number
  - name: status
    dataType: integer
    mode: required

dataProfile:
  [data-profile-data]

queryUsage:
  [top-fields and other usage aggregations]

descriptions:
  description: <generated table description>
  fields:
  - name: id
    description: ...
  - name: status
    description: ...

queries:
  queries:
  - description: ...
    sql: |
      [SQL Text]

links:
  definition:
    target: ecommerce.transaction
```

**workspace/catalog/ecommerce-prod.ecommerce-dataset/orders.overview.md**
```
---
userManaged: true
links:
- title: <title>
  url: <url>
---
[overview.content]
```

**workspace/catalog/ecommerce-prod.ecommerce-dataset/orders.guidelines.md**
```markdown
---
userManaged: false
---
[guidelines.instructions]
```

### Knowledge Base

Assume the user is managing a knowledge base of information managed in a hierarchical
structure. Or alternatively, an agent is building this knowledge base from previously
aggregated or enriched metadata.

**workspace/catalog.yaml**
```yaml
scope: kb.ecommerce-prod.global.mbr-kb

snapshot:
  entries:
  - document
  aspects:
  - overview

publishing:
  aspects:
  - overview
```

**workspace/catalog/products.md**
```markdown
---
id: products
type: document
createTime: <createTime>
updateTime: <updateTime>

resource:
  name: products
  displayName: Products
  description: ...
---
# Products Data

This is the markdown body representing `overview.content` for the document.
```

**workspace/catalog/playbooks/mbr.md**
```markdown
---
id: playbooks/mbr
type: document

resource:
  name: playbooks/mbr
  displayName: MBR Playbook
  description: ...
---
# Ecommerce Monthly Business Review Playbook

[overview.content]
```


================================================
FILE: toolbox\mdcode\docs\design.md
================================================
# Metadata as Code Design Specification

This document details the overall design for the Metadata as Code project, factoring in the sub-system breakdown, API surfaces, CLI and MCP structures, technical stack, and key technical decisions. It builds upon the initial context provided in:

- [readme.md](../README.md)
- [concept.md](concept.md)
- [spec.md](spec.md)
- [plan.md](plan.md)

## 1. Sub-system / Module Breakdown

The system is divided into two public interface layers and a core library foundation.

### 1.1 Public Interfaces
- **CLI (`kcmd`)**: A single binary built using **Bun** that handles user interaction and parses commands using the `cac` framework.
- **MCP Server**: Hosted within the same `kcmd` binary, triggered by the `mcp` command argument (e.g., `kcmd mcp`). It exposes sync operations as tools to external agents.

### 1.2 Core Library Foundation (`kc-mac`)
The brain of the operation, containing the following modules:
- **`CatalogManifest`**: Handles loading, validating, and parsing the `catalog.yaml` project manifest, including parsing the unified `scope` specification (which determines the layout style).
- **`CatalogSnapshot`**: Manages local catalog entries and coordinates high-level read/write/list logic by delegating physical file representation and layout-specific tasks to `CatalogLayout`.
- **`CatalogLayout`**: Abstract class defining the operations to list, read, write, and delete files on the local disk. Standard and Wiki layouts implement this interface, automatically selected based on the scope type.
- **`CatalogSync`**: Orchestrates pull and push sync directions, handles pagination, fail-fast conflict detection, and manages tool state in a separate state file (`.catalog.state`).
- **`CatalogService`**: Wrapper for raw HTTP API calls, handling Application Default Credentials (ADC) auth and LRO polling.

## 2. Project Directory Structure

This section describes the directory structure for the project source code, reflecting the merge of interfaces, TypeScript library naming for future expansion, and the `gcp` subfolder for API functionality:

- **`src/`**
  - **`tool/`**: The unified layer for CLI and MCP.
    - **`main.ts`**: The entry point for the unified binary (`kcmd`). Handles argument parsing to direct to either CLI commands or MCP server. Defines/handles CLI surface.
    - **`commands.ts`**: Contains functionality for CLI commands.
    - **`mcp.ts`**: Contains functionality for the MCP server and tools.
  - **`libts/`**: The TypeScript core library.
    - **`index.ts`**: The main entry point exporting public APIs.
    - **`manifest.ts`**: Contains the `CatalogManifest` class logic.
    - **`layout.ts`**: Contains the `CatalogLayout` interface.
    - **`source.ts`**: Contains the `CatalogSource` interface.
    - **`snapshot.ts`**: Contains the `CatalogSnapshot` class logic.
    - **`sync.ts`**: Contains the `CatalogSync` class logic.
    - **`gcp/`**: Subfolder for GCP-specific functionality.
      - **`api.ts`**: Contains the `Api` base class with polling and retries.
      - **`context.ts`**: Contains the `ApiContext` class.
      - **`dataplex.ts`**: Contains the API client for Knowledge Catalog.
    - **`sources/`**: Subfolder containing source-specific implementations.
      - **`entrygroup.ts`**: Encapsulates behavior for Dataplex EntryGroups.
      - **`bq-dataset.ts`**: Encapsulates behavior for BigQuery datasets.
      - **`wiki.ts`**: Encapsulates behavior for Wikis.
    - **`layouts/`**: Subfolder containing layout-specific implementations.
      - **`standard.ts`**: Encapsulates behavior for Standard layout.
      - **`documents.ts`**: Encapsulates behavior for Wiki layout.

## 3. Public API, CLI, and MCP Structure

### 2.1 Library API Surface

**`CatalogManifest`**
- `static initWithEntryGroup(entryGroup: string, ctx: ApiContext): CatalogManifest`
- `static initWithBigQuery(dataset: string, ctx: ApiContext): CatalogManifest`
- `static initWithKB(kb: string, ctx: ApiContext): CatalogManifest`
- `static load(path: string, ctx: ApiContext): Promise<CatalogManifest>`
- `save(path: string): void`

**`CatalogLayout` (Abstract Class)**
- `abstract list(snapshotPath: string): Promise<string[]>`
- `abstract readEntry(snapshotPath: string, entryId: string): Promise<Entry>`
- `abstract writeEntry(snapshotPath: string, entryId: string, entry: Entry): Promise<void>`
- `abstract deleteEntry(snapshotPath: string, entryId: string): Promise<void>`

**`StandardLayout` (extends `CatalogLayout`)**
- Implements filesystem storage using a main `.yaml` file per entry and optional sidecar `.overview.md` files for unstructured aspects.

**`WikiLayout` (extends `CatalogLayout`)**
- Implements filesystem storage using a single `.md` file per entry with the metadata in the YAML frontmatter and the overview content as the markdown body.

**`CatalogSnapshot`**
- `static fromPath(path: string): CatalogSnapshot`
- `layout: CatalogLayout`: The layout strategy instance initialized based on the manifest scope.
- `list(): Promise<string[]>`: Returns a list of entry IDs in the snapshot by delegating to `layout.list(snapshotPath)`.
- `lookupEntry(entryId: string): Promise<Entry>`: Returns structured metadata for a specific entry by delegating to `layout.readEntry(snapshotPath, entryId)`.
- `updateEntry(entryId: string, updates: Record<string, any>): Promise<void>`: Merges updates into the entry metadata and aspect values, and commits the result via `layout.writeEntry(snapshotPath, entryId, entry)`.
- `createEntry(entryId: string, data: Entry): Promise<void>`: Creates an entry locally by delegating to `layout.writeEntry(snapshotPath, entryId, data)`.
- `deleteEntry(entryId: string): Promise<void>`: Deletes files associated with the entry by delegating to `layout.deleteEntry(snapshotPath, entryId)`.

**`CatalogSync`**
- `pull(): Promise<SyncResult>`
- `push(options?: { force?: boolean, validateOnly?: boolean }): Promise<SyncResult>`
- `validate(): Promise<ValidationResult>`: Fetches fresh definitions and performs pre-push validation.
- `status(): Promise<StatusResult>`: Checks for local modifications against the saved checksum state.

### 2.2 CLI Command Structure (`kcmd`)

- `kcmd init [--bigquery-dataset <id>] [--entry-group <id>] [--kb <id>]`
- `kcmd pull [--dry-run]`
- `kcmd push [--dry-run] [--force] [--validate-only]`
- `kcmd status`
- `kcmd mcp` (Triggers the MCP server)

### 2.3 MCP Tools and Params

- **`pull`**: Params: None.
- **`push`**: Params: `force?: boolean`.
- **`list-entries`**: Params: None.
- **`lookup-entry`**: Params: `entryId: string`.
- **`update-entry`**: Params: `entryId: string`, and key/value pairs for each field and aspect being updated.

## 4. API Client Design & Context

### 3.1 `ApiContext` (Class)
- `project: string`
- `token: string`
- `static default(): Promise<ApiContext>`: Uses `gcloud` to read project and token defaults.
- `refresh(): Promise<void>`: Updates token via `gcloud auth print-access-token`.

### 3.2 `Api` (Base Class)
- `constructor(endpoint: string, pathPrefix: string, context: ApiContext)`
- **Methods**: `get`, `post`, `patch`, `delete` take relative resource names.
- **Polishing**:
  - Fully encapsulates Long Running Operation (LRO) polling and returns only the completed result or error.
  - Implements transient error retries with exponential backoff (configurable defaults).
  - Automatically calls `context.refresh()` on token expiry (401).

## 5. Technical Stack and Key Dependencies

- **Language**: TypeScript.
- **Runtime/Build**: **Bun** for standalone binary creation and fast startup.
- **Frameworks & Libraries**:
  - **CLI**: `cac` (minimal size).
  - **Schema Validation**: `zod` (TypeScript-first).
  - **Parsing**: `yaml`.
  - **HTTP Client**: Native `fetch` (in Bun) or `node-fetch` fallback.

## 6. Key Technical Decisions

1.  **Bun Unified Binary**: The CLI and MCP server will be distributed as the same single binary to reduce complexity for the end-user.
2.  **Thin HTTP Client**: We use native raw HTTP requests rather than Google SDKs to control code execution path and support environment workarounds.
3.  **Validation Before Action**: Fresh type definitions are pulled at publishing time, and full validation completes *before* the first change payload hits the cloud.
4.  **Fail-Fast Overwrites**: Overwrite protection fails fast when remote drift is identified.


================================================
FILE: toolbox\mdcode\docs\plan.md
================================================
# Phased Delivery Plan

This document outlines the phased delivery plan for building the Metadata as Code foundation. 

## Phase 1: MVP - Read-Only Snapshot & Consumption
Establish the local file structure by pulling metadata using the TypeScript library, and enable reading that snapshot via MCP. Support early distribution.

*   **Key Features & Work:**
    *   **Library (TS)**: Fetch metadata for a BigQuery dataset or Dataplex EntryGroup; create local directory structure mirroring the resource hierarchy; generate main YAML file per entry (Standard layout); paginated pull; ADC authentication.
    *   **CLI (TS-based)**: Implement `kcmd init` and a basic read-only `kcmd pull`.
    *   **MCP (TS-based)**: Implement an MCP server with tools: `list-entries` and `lookup-entry` (reading from the local snapshot).
    *   **Distribution**: Support installation from source repository or local package for early access.
    *   **Testing**: Implement test cases for snapshot creation and directory layout for BigQuery datasets and EntryGroups.

## Phase 2: MVP - Basic Push (Publish)
Complete the bi-directional sync by enabling local modifications to be pushed back to the catalog using the TypeScript library.

*   **Key Features & Work:**
    *   **Library (TS)**: Read local YAML files and reconstruct API payloads; push updates individually per entry; support parsing and filtering via `publishing` configuration in `catalog.yaml`.
    *   **CLI (TS-based)**: Implement a basic `kcmd push`.
    *   **Distribution**: Update early access packages with push capability.
    *   **Testing**: Implement test cases for basic push operation and publishing configuration filtering.

## Phase 3: Enhanced Representation
Optimize the file format for human and agent editing. Update interfaces to support the new formats and layout options.

*   **Key Features & Work:**
    *   **Library (TS)**: Support `kb` scope type alongside `bq-dataset` and `entryGroup` in `catalog.yaml`; implement the `CatalogLayout` abstraction (`layout.ts`) and its concrete layouts (`StandardLayout` and `DocumentsLayout`); refactor `CatalogSnapshot` to delegate filesystem list, read, and write operations to the active layout strategy (automatically instantiated based on the scope); support type aliases in `catalog.yaml`, including built-in aliases for built-in types.
    *   **CLI (TS-based)**: Update `pull` and `push` operations to handle sidecars, aliases, and different layout scopes correctly.
    *   **MCP (TS-based)**: Ensure `list-entries` and `lookup-entry` tools correctly handle layouts (by delegating via `CatalogSnapshot` to `CatalogLayout` strategy).
    *   **Testing**: Implement test cases for format mapping, including Standard layout (YAML + sidecars) and Documents layout (Markdown + frontmatter), and validate that `CatalogSnapshot` interacts correctly with both layouts based on the active scope type. Implement test cases for aliases.

## Phase 4: Robust Sync and State Management
Ensure data integrity and efficient updates.

*   **Key Features & Work:**
    *   **Library (TS)**: Store checksums of local state in a separate file; use checksums to detect changes; treat missing files as intent to delete; fail fast on remote modification; fetch type definitions dynamically for validation.
    *   **CLI (TS-based)**: Add `kcmd status` to check for local modifications; update `kcmd push` to use checksums and report conflicts; support **force override** flag; implement `--dry-run` option for `pull` and `push` commands.
    *   **Testing**: Implement test cases for checksum validation, intent to delete, validation against dynamic schema, and `--dry-run` behavior.

## Phase 5: Future / Other
Other workstreams to consider.

*   **Key Features & Work:**
    *   **Library (TS)**: Support for EntryLinks
    *   **Python Library**: Implement the equivalent library in Python to support Python-based agents and workflows.
    *   **MCP (TS-based)**: Implement `pull` and `push` tools in the MCP server.


================================================
FILE: toolbox\mdcode\docs\spec.md
================================================
# Metadata as Code Specification

This specification builds upon the concepts in `docs/concept.md` and `readme.md`, detailing the agreed-upon design decisions, critical user journeys, glossary, and test cases for the Metadata as Code project.

## 1. Overview

Metadata as Code provides a source code artifact-based UX for metadata management and context engineering in Knowledge Catalog (Dataplex). It is designed for agent builders, data stewards, data producers, and AI agents to author, manage, and enrich metadata using developer-friendly workflows with version control and CI/CD.

### 1.1. Primary Use Case
The primary driver for the initial design is the **Knowledge Catalog Enrichment Agent**, focusing on automating metadata curation with human-in-the-loop review. However, the library is designed to be flexible and usable by any developer building agents or custom tools.

## 2. Glossary of Terms

To ensure consistency, the following terms are used:

*   **Catalog Snapshot**: A directory on the local filesystem containing a subset of Knowledge Catalog metadata.
*   **Manifest (catalog.yaml)**: A file in the snapshot root that captures sync configuration and directives.
*   **Scope**: A unified identifier format (`<type>.<name>`) that establishes the single target resource for the snapshot in the manifest.
*   **Standard Layout**: A hybrid disk organization automatically used for `bq-dataset` and `entryGroup` scopes where metadata structure is stored in a YAML file per entry, and unstructured aspects are placed in sidecar Markdown files.
*   **Documents Layout**: A Markdown-only disk organization automatically used for `kb` scopes where the main entry is represented as a single Markdown file, with metadata structured in the YAML frontmatter and the main overview aspect promoted to the Markdown body.
*   **Entry**: A representation of a resource (like a table or dataset) in the Knowledge Catalog.
*   **Aspect**: A specific type of metadata attached to an entry (e.g., description, profile).
*   **Sidecar File**: An auxiliary file (e.g., Markdown) used for aspects with unstructured text.
*   **Source**: A backend metadata repository referenced by a scope (e.g., Dataplex EntryGroup or BigQuery Dataset).

## 3. Key Design Decisions

The following core decisions have been established:

### 3.1. Metadata Representation
The disk organization layout is automatically determined by the scope type:
*   **Standard Layout** (used for `bq-dataset` and `entryGroup` scopes): Structured data (entry metadata, resource info, aspects like schema and profile) is stored in a main YAML file per entry, while unstructured rich-text fields (like Overview) are stored in dedicated sidecar Markdown files.
*   **Documents Layout** (used for `kb` scopes): Structured metadata is stored within the YAML frontmatter of a single `.md` file per entry, and the primary unstructured aspect (`overview.content`) is promoted to serve as the main Markdown body of that file. Other unstructured aspects may still use sidecars.

### 3.2. Synchronization Model
We will support a bi-directional sync (Pull and Push) with the following rules:
*   **Publishing is a subset** of the snapshot list.
*   While required aspects may be implicitly included in the snapshot, all entry types and aspect types that should be published **must be explicitly listed** in the manifest.

### 3.3. Conflict Handling
To prevent data loss during a `push` operation:
*   The tool will **fail fast** if it detects that the metadata has been modified in the catalog in the interim.
*   It will abort the push, report the conflict, and require a `pull` to resolve.
*   A **force override** option will be provided to bypass this check if necessary.

### 3.4. Aliases Strategy
To maintain consistency and steer users toward simpler names:
*   If an alias is defined in `catalog.yaml`, it **must be used consistently** in metadata files.
*   Otherwise, the full reference takes the form of `<projectId>.<locationId>.<resourceId>`.

### 3.5. Scale and Performance
*   **Paginated Pull**: Pull operations will be paginated to handle large resources.
*   **Individual Push**: Push operations will be performed individually per entry for now, with bulk import operations considered later.
*   **Scale Assumption**: Users are managing this as code artifacts in a repository and should not be operating at hyperscale/unbounded scale in this workflow.

### 3.6. Extensibility and Validation
*   **Dynamic Schema Fetching**: The tool will fetch type definitions from the Catalog service dynamically to validate local files against the actual registered schema.

### 3.7. Checksum Storage
*   **Separate State File**: Checksums for entries and aspects will be stored in a separate state file (e.g., `.catalog.state`) to separate user content from tool state.

### 3.8. Sync Deletions
*   **Intent to Delete**: If an entry type is listed in the publish config, treat missing files compared to state as intent to delete.
*   **Skip for Managed Entries**: Deletions are skipped for Dataplex managed entries (like BigQuery resources) which cannot be deleted.

### 3.9. Unified Scope Mapping
*   **Single String Identifier**: Snapshot resources are specified using a unified `scope` property formatted as `<type>.<name>` (e.g. `bq-dataset.<id>` or `entryGroup.<id>`). This unifies target definition logic and enables future extension to other resource types easily.

## 4. Critical User Journeys (CUJs)

We focus on the following core journeys:

*   **CUJ 1: Initial Setup and Snapshot Creation**
    User runs `kcmd init` to create a local snapshot of a BigQuery dataset or Dataplex EntryGroup.
*   **CUJ 2: Metadata Sync (Pull and Push)**
    User pulls updates from the catalog service, makes modifications to local files, and pushes changes back to the catalog.

*Note: Advanced workflows like Automated Curation with Human Review (CUJ 3 candidates) are built on top of this foundation using external tools (IDE, git, custom agents) and are out of scope for this base library.*

## 5. Key Test Cases

To validate the design, we will implement test cases covering the following areas, including user feedback:

### 5.1. Resource Types
*   **Snapshots for different resource types**: BigQuery datasets and Dataplex EntryGroups.
*   **Entry and aspect varieties**: Both Dataplex-defined entries/aspects (where required aspects are read-only) and user-defined ones (where required aspects are modifiable).

### 5.2. Format Mapping
*   **Standard Layout** (for `bq-dataset` and `entryGroup` scopes): Aspects managed correctly within the main YAML file, and unstructured text mapped correctly to/from sidecar Markdown files.
*   **Documents Layout** (for `kb` scopes): Metadata mapped correctly to/from YAML frontmatter, and the `overview.content` aspect promoted to/from the Markdown file body.

### 5.3. Directory Organization
*   **BQ Resource Hierarchy**: Testing layout for BigQuery datasets.
*   **EntryGroup Hierarchy**: Testing layout for EntryGroups based on path segments in the entry ID.

### 5.4. Data Integrity
*   **Checksum validation**: Verifying that checksums are updated and validated for entries and aspects during both pull and push operations to detect conflicts.


================================================
FILE: toolbox\mdcode\README.md
================================================
# Metadata as Code

Metadata as Code is a Knowledge Catalog (Dataplex) provides data stewards and data producers and AI agents with a source code artifact-based UX for metadata management and context engineering.

Users and agents can author, manage, and enrich metadata artifacts using developer-friendly workflows with version control and CI/CD. It provides a standard metadata format can be used by a variety of tools and agents.

More details are in the [docs/concept.md](docs/concept.md).

## Key Features

* Intuitive human and agent-friendly representation of metadata as source code in YAML and markdown files. Artifacts are organized in a hierarchical manner mirroring the resource hierarchy of data and metadata assets.
* Bi-directional sync between local workspace and catalog service.
* Support for both 1st party and 3rd party metadata constructs.
* Distributed as a TypeScript and Python libraries, a CLI tool (kcmd), and an MCP server for use in a variety of applications, agents and workflows.

## Metadata Artifacts

### Directory Layout
Metadata is organized within a directory representing a resource such as a BigQuery Dataset, Dataplex EntryGroup etc.

```
path/to/root/
├── catalog.yaml                       # Manifest and config directives
└── catalog/                           # Contains the metadata snapshot
    └── <dir1>/
        └── <entry-id1>.yaml           # Entry
        └── <dir2>/
            ├── <entry-id2>.yaml       # Entry with sidecar markdown
            └── <entry-id2>.aspect.md  # files
```

## Catalog Snapshot Files

### Manifest File
**catalog/catalog.yaml**
```yaml
scope: bq-dataset.prod-data.ecommerce

aliases:
  ca-guidelines:
    aspect: data-agents-project.global.ca-guidelines
  ecommerce:
    aspect: data-agents-project.global.ecommerce

snapshot:
  entries:
    - bigquery-table
    - bigquery-view
    - entry-group
  aspects:
    - overview
    - descriptions

publishing:
  aspects:
    - overview
    - descriptions
```

## Entry YAML File
**catalog/prod-data.ecommerce/products.yaml**
```yaml
id: products
type: bigquery-table

resource:
  name: projects/prod-data/datasets/ecommerce/tables/products
  displayName: Products Table
  description: All products in the catalog
  labels:
    env: prod
  createTime: 2026-04-23T00:44:03Z
  updateTime: 2026-04-23T00:44:03Z

schema:
  ...

contacts:
  ...
```

## Entry Sidecar Markdown File
**catalog/prod-data.ecommerce/products.overview.md**

```markdown
---
userManaged: true
links:
  ...
---
[overview.content]
```

## Usage

### Library

You can use the `kcmd` library to programmatically interact with the catalog metadata.

```bash
npm install kcmd
```

```typescript
import * as kcmd from 'kcmd';

// Creating a catalog manifest from scratch
const manifest = new kcmd.CatalogManifest(...);
manifest.save('/path/to/root');

// Loading a catalog snapshot from the filesystem
const snapshot = kcmd.CatalogSnapshot.fromPath('/path/to/root');

// Pulling the latest metadata from the Catalog service.
const pullResult = await snapshot.pull();
if (pullResult.success) {
  console.log('Metadata pulled successfully');
}
else {
  console.error('Metadata pull failed:', pullResult.error);
}

// Pushing the modified metadata to the Catalog service.
const pushResult = await snapshot.push();
if (pushResult.success) {
  console.log('Metadata pushed successfully');
}
else {
  console.error('Metadata push failed:', pushResult.error);
}
```

### CLI

The package provides the `kcmd` CLI tool. This is distributed as a standalone binary.

```bash
# Initialize a new catalog snapshot for a bigquery dataset
kcmd init --bigquery-dataset <projectId>.<datasetId>

# Initialize a new catalog snapshot for multiple bigquery datasets
kcmd init --bigquery-dataset <projectId>.<datasetId1> --bigquery-dataset <projectId>.<datasetId2>

# Initialize a new catalog snapshot for a bigquery dataset with specific types
kcmd init --bigquery-dataset <projectId>.<datasetId> \
  --entry bigquery-table --entry bigquery-view \
  --aspect overview --aspect description

# Initialize a new catalog snapshot for a custom EntryGroup
kcmd init --entry-group <projectId>.<locationId>.<entryGroupId>

# Pull the latest catalog snapshot from the Knowledge Catalog service
# Reports any conflicts if there are pending changes that have not been
# pushed to the catalog.
# Supports dry run with the --dry-run flag.
kcmd pull

# Check for local modifications
kcmd status

# Push local changes to the Knowledge Catalog service. Only pushes changes
# made since the last pull, and if that metadata has not been modified within
# catalog in the interim.
# Supports dry run with the --dry-run flag.
kcmd push
```

NOTE: The CLI uses `gcloud` to obtain authentication tokens, so ensure you are authenticated via `gcloud auth application-default login`.

### MCP Server

To use the Metadata as Code tools as MCP tools in an agentic system such as the Gemini CLI, add the following to your MCP settings file:

```json
{
  "mcpServers": {
    "kc-mac": {
      "command": "kcmd",
      "args": ["mcp", "--path", "/path/to/root"]
    }
  }
}
```

The server offers the following tools: 

| Tool             | Description                                           |
|------------------|-------------------------------------------------------|
| pull             | Pull the latest metadata from the Catalog service     |
| push             | Push the modified metadata to the Catalog service     |
| list-entries     | List entries in the catalog snapshot                  |
| lookup-entry     | Lookup an entry and its metadata from the snapshot    |
| modify-entry     | Modify an entry and its metadata in the snapshot      |

NOTE: The server uses `gcloud` to obtain authentication tokens, so ensure you are authenticated via `gcloud auth application-default login`.

## Developer Workflow

### Setup

```bash
git clone https://github.com/googlecloudplatform/knowledge-catalog
cd toolbox/mdcode
npm install
```

### Build

```bash
npm run build
```

### Test

```bash
npm run test
```


================================================
FILE: toolbox\mdcode\src\libts\gcp\api.ts
================================================
// API Client base class
//

import * as context from './context';


export interface ApiResult<T> {
  status: number;
  result?: T;
  message?: string;
}


export class ApiClient {
  private readonly _endpoint: string;
  private readonly _pathPrefix: string;
  private readonly _context: context.ApiContext;

  constructor(endpoint: string, pathPrefix: string, context: context.ApiContext) {
    this._endpoint = endpoint;
    this._pathPrefix = pathPrefix;
    this._context = context;
  }

  get context(): context.ApiContext {
    return this._context;
  }

  async _get<T>(resourceName: string,
                 queryParams?: Record<string, any>): Promise<ApiResult<T>> {
    const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`;
    return this._requestRetry('GET', url, queryParams);
  }

  async _post<T>(resourceName: string,
                 body: any,
                 queryParams?: Record<string, any>): Promise<ApiResult<T>> {
    const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`;
    return this._requestRetry('POST', url, queryParams, body);
  }

  async _patch<T>(resourceName: string,
                  body: any,
                  queryParams?: Record<string, any>): Promise<ApiResult<T>> {
    const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`;
    return this._requestRetry('PATCH', url, queryParams, body);
  }

  private async _requestRetry<T>(method: string,
                                 url: string,
                                 queryParams?: Record<string, any>,
                                 body?: any): Promise<ApiResult<T>> {
    if (queryParams) {
      const params = new URLSearchParams();
      for (const [key, value] of Object.entries(queryParams)) {
        if (value !== undefined) {
          if (Array.isArray(value)) {
            value.forEach(v => params.append(key, String(v)));
          }
          else {
            params.append(key, String(value));
          }
        }
      }
      const queryString = params.toString();
      if (queryString) {
        url += `?${queryString}`;
      }
    }

    this._context.log(`${method} ${url}{body ? '\n' : ''}`, body);

    let response = await this._requestCore(url, { method, body });
    if (response.status === 401) {
      this.context.refresh();
      response = await this._requestCore(url, { method, body });
    }

    const result: ApiResult<T> = { status: response.status };
    if (!response.ok) {
      result.message = await response.text();
    }
    else {
      result.result = await response.json() as T;
    }

    this._context.log(`${response.status}:${result.message ?? ''}\n`, result.result);
    return result;
  }

  private async _requestCore(url: string, options: any): Promise<Response> {
    const headers = {
      ...options.headers,
      'Authorization': `Bearer ${this._context.token}`,
      'Content-Type': 'application/json'
    };

    return fetch(url, {
      ...options,
      headers,
      body: options.body ? JSON.stringify(options.body) : undefined,
    });
  }
}


================================================
FILE: toolbox\mdcode\src\libts\gcp\bigquery.ts
================================================
// API client for BigQuery
//

import * as api from './api';
import * as context from './context';


export interface Dataset {
  id: string;
  datasetReference: {
    projectId: string;
    datasetId: string;
  };
  location: string;
  [key: string]: any;
}

export interface Table {
  id: string;
  tableReference: {
    projectId: string;
    datasetId: string;
    tableId: string;
  };
  [key: string]: any;
}

interface TableList {
  tables: Table[];
  nextPageToken?: string;
}


export class BigQueryClient extends api.ApiClient {

  constructor(ctx: context.ApiContext) {
    super('https://bigquery.googleapis.com', 'bigquery/v2', ctx);
  }

  async getDataset(project: string, dataset: string): Promise<api.ApiResult<Dataset>> {
    const name = `projects/${project}/datasets/${dataset}`;
    const params: Record<string, any> = { datasetView: 'METADATA' };

    return await this._get(name, params);
  }

  async *listTables(project: string, dataset: string): AsyncGenerator<Table> {
    const name = `projects/${project}/datasets/${dataset}/tables`;

    let pageToken: string | undefined = undefined;
    do {
      const params: Record<string, any> = { maxResults: 500 };
      if (pageToken) {
        params.pageToken = pageToken;
      }

      const res = await this._get<TableList>(name, params);
      if (res.status !== 200) {
        throw new Error(`Failed to list tables: ${res.message || res.status}`);
      }

      const tables = res.result?.tables || [];
      for (const table of tables) {
        yield table;
      }

      pageToken = res.result?.nextPageToken;
    } while (pageToken);
  }
}


================================================
FILE: toolbox\mdcode\src\libts\gcp\context.ts
================================================
// Context variables for GCP API requests
//

import * as cp from 'child_process';

const GCLOUD_PROJECT_CMD = 'gcloud -q config get-value project';
const GCLOUD_LOCATION_CMD = 'gcloud -q config get-value compute/region';
const GCLOUD_TOKEN_CMD = 'gcloud -q auth application-default print-access-token';

export class ApiContext {
  readonly project: string;
  readonly location: string;
  private _token: string;

  constructor(project: string, location: string, token: string) {
    this.project = project;
    this.location = location;
    this._token = token;
  }

  get token(): string {
    return this._token;
  }

  log(message: string, data?: any) {
    if (process.env.GCP_LOG) {
      console.log(`[GCP_LOG] ${message}`, data ? JSON.stringify(data) : '');
    }
  }

  static default(): ApiContext {
    // Creates an ApiContext instance using gcloud configuration

    const project = cp.execSync(GCLOUD_PROJECT_CMD).toString().trim();
    const location = cp.execSync(GCLOUD_LOCATION_CMD).toString().trim();
    const token = cp.execSync(GCLOUD_TOKEN_CMD).toString().trim();
    if (!project || !location || !token) {
      throw new Error('Unable to retrieve project, location, or token. Ensure gcloud is configured.');
    }

    return new ApiContext(project, location, token);
  }

  refresh() {
    this._token = cp.execSync(GCLOUD_TOKEN_CMD).toString().trim();
  }
}


================================================
FILE: toolbox\mdcode\src\libts\gcp\crm.ts
================================================
// API client for Cloud Resource Manager
//

import * as api from './api';
import * as context from './context';


export interface Project {
  name: string;
  projectId: string;
  [key: string]: any;
}


const PROJECT_NUM_TO_ID_CACHE = new Map<string, string>();
PROJECT_NUM_TO_ID_CACHE.set('655216118709', 'dataplex-types');


export class ResourceManagerClient extends api.ApiClient {

  constructor(ctx: context.ApiContext) {
    super('https://cloudresourcemanager.googleapis.com', 'v3', ctx);
  }

  async getProject(project: string): Promise<api.ApiResult<Project>> {
    const name = `projects/${project}`;
    return await this._get(name);
  }
}

export async function fixProject(resource: string, ctx: context.ApiContext): Promise<string> {
  // projects/<project_id> or projects/<project_number> -> projects/<project_id>

  const parts = resource.split('/');
  if (/^\d+$/.test(parts[1])) {
    let id = PROJECT_NUM_TO_ID_CACHE.get(parts[1]);
    if (!id) {
      const res = await new ResourceManagerClient(ctx).getProject(parts[1]);
      id = res.status == 200 ? res.result?.projectId : '';
    }

    if (id) {
      PROJECT_NUM_TO_ID_CACHE.set(parts[1], id);
      parts[1] = id;
    }
    resource = parts.join('/');
  }

  return resource;
}


================================================
FILE: toolbox\mdcode\src\libts\gcp\dataplex.ts
================================================
// API client for Knowledge Catalog (Dataplex)
//

import * as api from './api';
import * as context from './context';
import * as crm from './crm';


export interface EntryGroup {
  name: string;
  [key: string]: any;
}

export interface EntryType {
  name: string;
  requiredAspects: { type: string; }[];
  [key: string]: any;
}

export interface AspectType {
  name: string;
  [key: string]: any;
}

export interface Aspect {
  aspectType?: string;
  data?: Record<string, any>;
}

export interface Entry {
  name: string;
  entryType: string;
  parentEntry?: string;
  createTime?: string;
  updateTime?: string;
  entrySource?: {
    resource?: string;
    ancestors?: {
      name: string;
      type: string;
    }[];
    displayName?: string;
    description?: string;
    labels?: Record<string, string>;
    location?: string;
    createTime?: string;
    updateTime?: string; 
  };
  aspects?: Record<string, Aspect>;
}

interface EntryList {
  entries: Entry[];
  nextPageToken?: string;
}


export class CatalogClient extends api.ApiClient {

  constructor(ctx: context.ApiContext) {
    super('https://dataplex.googleapis.com', 'v1', ctx);
  }

  async getEntryGroup(project: string, location: string,
                      entryGroup: string): Promise<api.ApiResult<EntryGroup>> {
    const name = catalogContainer(project, location, entryGroup);
    return await this._get(name);
  }

  async getEntryType(project: string, location: string,
                     type: string): Promise<api.ApiResult<EntryType>> {
    const name = `${catalogContainer(project, location)}/entryTypes/${type}`;
    return await this._get(name);
  }

  async getAspectType(project: string, location: string,
                      type: string): Promise<api.ApiResult<AspectType>> {
    const name = `${catalogContainer(project, location)}/aspectTypes/${type}`;
    return await this._get(name);
  }

  async getEntry(project: string, location: string, entryGroup: string, entry: string,
                 aspects?: string[]): Promise<api.ApiResult<Entry>> {
    const name = `${catalogContainer(project, location, entryGroup)}/entries/${entry}`;
    const params: Record<string, any> = { view: 'BASIC' };
    if (aspects && aspects.length) {
      params.view = 'CUSTOM';
      params.aspectTypes = aspects;
    }

    const res = await this._get<Entry>(name, params);
    if (res.status == 200 && res.result) {
      await _fixEntry(res.result, this.context);
    }

    return res;
  }

  async lookupEntry(project: string, location: string, name: string,
                    aspects?: string[]): Promise<api.ApiResult<Entry>> {
    const container = `${catalogContainer(project, location)}:lookupEntry`;
    const params: Record<string, any> = { entry: name, view: 'BASIC' };
    if (aspects && aspects.length) {
      params.view = 'CUSTOM';
      params.aspectTypes = aspects;
    }

    const res = await this._get<Entry>(container, params);
    if (res.status == 200 && res.result) {
      await _fixEntry(res.result, this.context);
    }

    return res;
  }

  async modifyEntry(project: string, location: string, entry: Entry,
                    updateMask?: string[],
                    aspectKeys?: string[]): Promise<api.ApiResult<Entry>> {
    const container = `${catalogContainer(project, location)}:modifyEntry`;
    const body: Record<string, any> = {
      entry: entry,
      updateMask: updateMask ? updateMask.join(',') : undefined,
      aspectKeys: aspectKeys ?? undefined
    };

    const res = await this._post<Entry>(container, body);
    if (res.status == 200 && res.result) {
      await _fixEntry(res.result, this.context);
    }

    return res;
  }

  async updateEntry(entry: Entry,
                    updateMask?: string[],
                    aspectKeys?: string[]): Promise<api.ApiResult<Entry>> {
    const params: Record<string, any> = {};
    if (updateMask && updateMask.length) {
      params.updateMask = updateMask.join(',');
    }
    if (aspectKeys && aspectKeys.length) {
      params.aspectKeys = aspectKeys;
    }

    const res = await this._patch<Entry>(entry.name, entry, params);
    if (res.status == 200 && res.result) {
      await _fixEntry(res.result, this.context);
    }

    return res;
  }

  async *listEntries(project: string, location: string,
                     entryGroup: string): AsyncGenerator<Entry, void, unknown> {
    const parent = catalogContainer(project, location, entryGroup);
    const resourceName = `${parent}/entries`;

    let pageToken: string | undefined = undefined;
    do {
      const params: Record<string, string | number> = { pageSize: 1000 };
      if (pageToken) {
        params.pageToken = pageToken;
      }

      const res = await this._get<EntryList>(resourceName, params);
      if (res.status !== 200) {
        throw new Error(`Failed to list entries: ${res.message || res.status}`);
      }

      const entries = res.result?.entries || [];
      for (const entry of entries) {
        await _fixEntry(entry, this.context);
        yield entry;
      }

      pageToken = res.result?.nextPageToken;
    } while (pageToken);
  }

  async createEntry(project: string, location: string, entryGroup: string, 
                    entryId: string, entry?: Entry): Promise<api.ApiResult<Entry>> {
    const parent = catalogContainer(project, location, entryGroup);
    const resourceName = `${parent}/entries`;

    const params: Record<string, any> = { entryId };

    const res = await this._post<Entry>(resourceName, entry, params);
    
    if (res.status == 200 && res.result) {
      await _fixEntry(res.result, this.context);
    }

    return res;
  }

  async createEntryGroup(project: string, location: string, 
                         entryGroupId: string, entryGroup?: EntryGroup): Promise<api.ApiResult<EntryGroup>> {
    const parent = catalogContainer(project, location);
    const resourceName = `${parent}/entryGroups`;

    const params: Record<string, any> = { entryGroupId };

    const res = await this._post<EntryGroup>(resourceName, entryGroup, params);

    return res;
  }

}


// Fix all entries and aspects to consistently use project id. Its currently a mess with an
// inconsistent mix of project ids and unusable project numbers.
async function _fixEntry(entry: Entry, ctx: context.ApiContext): Promise<void> {
  entry.name = await crm.fixProject(entry.name, ctx);
  entry.entryType = await crm.fixProject(entry.entryType, ctx);
  if (entry.entrySource?.resource) {
    entry.entrySource.resource = await crm.fixProject(entry.entrySource.resource, ctx);
  }

  if (entry.aspects) {
    const fixedAspects: Record<string, Aspect> = {};
    for (const [aspectKey, aspectValue] of Object.entries(entry.aspects)) {
      let aspectType = '';
      if (!aspectValue || Object.keys(aspectValue).length) {
        aspectType = _typeRefToName(aspectKey, 'aspect');
      }
      else {
        aspectType = aspectValue['aspectType'] as string;
      }
      aspectType = await crm.fixProject(aspectType, ctx);

      fixedAspects[_nameToTypeRef(aspectType)] = {
        aspectType: aspectType,
        data: aspectValue['data'] ?? {}
      };
    }
    entry.aspects = fixedAspects;
  }
}

// Constructs canonical names for catalog container resources, identified by project, location and
// optionally, depending on use-case, the entry group.
export function catalogContainer(project: string, location: string, entryGroup: string=''): string {
  let container = `projects/${project}/locations/${location}`;
  if (entryGroup) {
    container += `/entryGroups/${entryGroup}`;
  }

  return container;
}

// Converts project.location.type to projects/${project}/locations/${location}/typeTypes/${type}
export function _typeRefToName(ref: string, type: string): string {
  const refParts = ref.split('.');
  if (refParts.length !== 3) {
    throw new Error(`Invalid type reference: ${ref}`);
  }
  return `projects/${refParts[0]}/locations/${refParts[1]}/${type}Types/${refParts[2]}`;
}

// Converts projects/${project}/locations/${location}/typeTypes/${type} -> project.location.type
export function _nameToTypeRef(name: string): string {
  const nameParts = name.split('/');
  if (nameParts.length < 6) {
    throw new Error(`Invalid type name: ${name}`);
  }
  return `${nameParts[1]}.${nameParts[3]}.${nameParts[5]}`;
}


================================================
FILE: toolbox\mdcode\src\libts\gcp\index.ts
================================================
// GCP Client Library and Helpers
//

export * from './context';
export * from './api';
export * from './dataplex';


================================================
FILE: toolbox\mdcode\src\libts\index.ts
================================================
export * from './manifest';
export * from './metadata';
export * from './snapshot';
export * from './sync';

export * as gcp from './gcp/context';
export * as dataplex from './gcp/dataplex';


================================================
FILE: toolbox\mdcode\src\libts\layout.ts
================================================
// Defines the Catalog metadata layout abstraction.
//

import * as md from './metadata';
import { StandardLayout } from './layouts/standard';
import { DocumentsLayout } from './layouts/documents';

export enum Layouts {
  STANDARD = 'standard',
  DOCUMENTS = 'documents'
}


export interface CatalogLayout {
  init(): Promise<void>;

  entryExists(name: string): boolean;
  listEntries(): string[];
  loadEntry(name: string): Promise<md.Entry>;
  saveEntry(name: string, entry: md.Entry): Promise<void>;
  deleteEntry(name: string): Promise<void>;
}


export function createLayout(layout: Layouts,
                             catalogPath: string): CatalogLayout {
  switch (layout) {
    case Layouts.STANDARD:
      return new StandardLayout(catalogPath);
    case Layouts.DOCUMENTS:
      return new DocumentsLayout(catalogPath);
    default:
      throw new Error(`Unknown layout type: ${layout}`);
  }
}


================================================
FILE: toolbox\mdcode\src\libts\layouts\documents.ts
================================================
// Implements the documents layout (markdown files in directory)
//

import * as fs from 'node:fs';
import * as glob from 'glob';
import * as path from 'node:path';
import * as yaml from 'yaml';
import * as md from '../metadata';
import { CatalogLayout } from '../layout';

const OVERVIEW_ASPECT_KEY = 'dataplex-types.global.overview';
const DEFAULT_ENTRY_TYPE = 'dataplex-types.global.generic';
const INDEX_NAME = 'index';


export class DocumentsLayout implements CatalogLayout {

  private _catalogPath: string = '';

  // Maps entry name -> local file path, or null for synthetic directory-index
  // entries that don't have a backing file on disk.
  private readonly _index = new Map<string, string | null>();

  constructor(catalogPath: string) {
    this._catalogPath = catalogPath;
  }

  async init(): Promise<void> {
    this._index.clear();

    if (!fs.existsSync(this._catalogPath)) {
      return;
    }

    const matches = await glob.glob('**/*.md', {
      cwd: this._catalogPath,
      absolute: true,
      nodir: true,
    });

    for (const localPath of matches) {
      const name = deriveEntryNameFromPath(localPath, this._catalogPath);
      this._index.set(name, localPath);
    }

    this._synthesizeIndexEntries();
  }

  entryExists(name: string): boolean {
    return this._index.has(name);
  }

  listEntries(): string[] {
    return Array.from(this._index.keys());
  }

  async loadEntry(name: string): Promise<md.Entry> {
    if (!this._index.has(name)) {
      throw new Error(`Entry not found: ${name}`);
    }
    const entryPath = this._index.get(name) ?? null;

    let parsed: md.Entry | null = null;
    let body = '';
    if (entryPath) {
      const content = await fs.promises.readFile(entryPath, 'utf8');
      const result = parseMarkdown(content);
      parsed = result.entry;
      body = result.body;
    }

    const entry: md.Entry = parsed ?? ({ type: DEFAULT_ENTRY_TYPE, resource: {} } as md.Entry);
    entry.name = name;

    entry.resource = entry.resource ?? {};
    const parentName = deriveParentLocalName(name);
    if (parentName !== undefined) {
      entry.resource.parent = parentName;
    } else {
      delete entry.resource.parent;
    }

    // Ensure the entry's type aspect is present — Dataplex create requires it.
    entry.aspects = entry.aspects ?? {};
    entry.aspects[entry.type] = entry.aspects[entry.type] ?? {};

    const bodyTrimmed = body.trim();
    if (bodyTrimmed) {
      if (!entry.aspects[OVERVIEW_ASPECT_KEY]) {
        entry.aspects[OVERVIEW_ASPECT_KEY] = {};
      }
      entry.aspects[OVERVIEW_ASPECT_KEY].content = bodyTrimmed;
      entry.aspects[OVERVIEW_ASPECT_KEY].contentType = 'MARKDOWN';
    }
    return entry;
  }

  async saveEntry(name: string, entry: md.Entry): Promise<void> {
    const entryPath = path.join(this._catalogPath, `${name}.md`);
    await fs.promises.mkdir(path.dirname(entryPath), { recursive: true });

    // Clone to avoid mutating original entry aspects
    const clonedEntry = JSON.parse(JSON.stringify(entry)) as md.Entry;
    let body = '';

    if (clonedEntry.aspects?.[OVERVIEW_ASPECT_KEY]) {
      const aspect = clonedEntry.aspects[OVERVIEW_ASPECT_KEY];
      if (aspect.content !== undefined) {
        body = aspect.content;
        delete aspect.content;
        delete aspect.contentType;
      }
    }

    const fileContent = toMarkdown(clonedEntry, body);

    await fs.promises.writeFile(entryPath, fileContent, 'utf8');
    this._index.set(name, entryPath);
    this._synthesizeIndexEntries();
  }

  async deleteEntry(name: string): Promise<void> {
    const entryPath = this._index.get(name);
    if (!entryPath) {
      throw new Error(`Entry not found: ${name}`);
    }
    if (!fs.existsSync(entryPath)) {
      throw new Error(`Entry not found: ${name}`);
    }

    await fs.promises.unlink(entryPath);
    this._index.delete(name);
    this._synthesizeIndexEntries();
  }

  // Ensure every directory that contains entries has an index entry. Directories
  // whose own index file is absent get a synthetic entry (no backing file).
  // Synthetic entries left behind by a previous synthesis pass are dropped if
  // their directory no longer has any other entries.
  private _synthesizeIndexEntries(): void {
    for (const [name, entryPath] of this._index) {
      if (entryPath === null) {
        this._index.delete(name);
      }
    }

    const dirs = new Set<string>();
    for (const name of this._index.keys()) {
      const segments = name.split('/');
      for (let i = 0; i < segments.length; i++) {
        dirs.add(segments.slice(0, i).join('/'));
      }
    }

    for (const dir of dirs) {
      const indexName = dir ? `${dir}/${INDEX_NAME}` : INDEX_NAME;
      if (!this._index.has(indexName)) {
        this._index.set(indexName, null);
      }
    }
  }
}

function deriveEntryNameFromPath(absolutePath: string, catalogPath: string): string {
  const rel = path.relative(catalogPath, absolutePath);
  return rel.replace(/\.md$/, '');
}

function deriveParentLocalName(name: string): string | undefined {
  const segments = name.split('/');
  const leaf = segments[segments.length - 1];

  // The root index entry has no parent.
  if (leaf === INDEX_NAME && segments.length === 1) {
    return undefined;
  }

  const parentDir = leaf === INDEX_NAME
    ? segments.slice(0, -2)
    : segments.slice(0, -1);

  return [...parentDir, INDEX_NAME].join('/');
}

export function parseMarkdown(content: string): { entry: md.Entry|null; body: string } {
  const lines = content.split(/\r?\n/);
  if (lines[0] !== '---') {
    return { entry: null, body: content };
  }
  const endIndex = lines.indexOf('---', 1);
  if (endIndex === -1) {
    return { entry: null, body: content };
  }

  const frontmatter = lines.slice(1, endIndex).join('\n');
  const metadata = yaml.parse(frontmatter);
  const body = lines.slice(endIndex + 1).join('\n');

  const entry = (metadata.catalogEntry ?? {}) as md.Entry;
  entry.type = (typeof metadata.type === 'string' && metadata.type.split('.').length === 3)
    ? metadata.type
    : DEFAULT_ENTRY_TYPE;
  entry.resource = entry.resource ?? {}
  entry.resource.displayName = metadata.title;
  entry.resource.description = metadata.description;
  if (metadata.tags) {
    entry.resource.labels = entry.resource.labels ?? {};
    for (const tag of metadata.tags) {
      entry.resource.labels[tag] = 'true';
    }
  }
  if (metadata.timeStamp) {
    entry.resource.updateTime = metadata.timeStamp;
    if (!entry.resource.createTime) {
      entry.resource.createTime = metadata.timeStamp;
    }
  }

  return { entry, body };
}

export function toMarkdown(entry: md.Entry, body: string): string {
  // Clone to be able to make modifications
  const entryClone = JSON.parse(JSON.stringify(entry)) as Record<string, any>;

  const tags = [];
  if (entry.resource.labels) {
    for (const [k, v] of Object.entries(entryClone.resource.labels ?? {})) {
      if (v == 'true') {
        tags.push(k);
      }
    }
  }

  const metadata = {
    type: entry.type,
    title: entry.resource.displayName ?? entry.resource.name,
    description: entry.resource.description ?? undefined,
    tags: tags.length ? tags : undefined,
    timeStamp: entry.resource.updateTime ?? entry.resource.createTime ?? undefined,
    catalogEntry: entryClone
  };

  delete entryClone.name;
  delete entryClone.resource.displayName;
  delete entryClone.resource.description;
  delete entryClone.resource.updateTime;
  delete entryClone.resource.createTime;
  delete entryClone.resource.parent;
  delete entryClone.type;
  for (const tag of tags) {
    delete entryClone.resource.labels[tag];
  }

  const frontmatter = yaml.stringify(metadata).trim();
  return `---\n${frontmatter}\n---\n${body}`;
}


================================================
FILE: toolbox\mdcode\src\libts\layouts\standard.ts
================================================
// Implements the standard layout (yaml files in directory)
//

import * as fs from 'node:fs';
import * as path from 'node:path';
import * as glob from 'glob';
import * as yaml from 'yaml';
import { CatalogLayout } from '../layout';
import * as md from '../metadata';


export class StandardLayout implements CatalogLayout {

  private readonly _catalogPath: string;

  private readonly _index = new Map<string, string>();

  constructor(catalogPath: string) {
    this._catalogPath = catalogPath;
  }

  async init(): Promise<void> {
    this._index.clear();

    if (!fs.existsSync(this._catalogPath)) {
      return;
    }

    const matches = await glob.glob('**/*.yaml', {
      cwd: this._catalogPath,
      absolute: true,
      nodir: true,
    });

    for (const localPath of matches) {
      try {
        const content = await fs.promises.readFile(localPath, 'utf8');
        const metadata = yaml.parse(content);
        if (metadata && metadata.name) {
          this._index.set(metadata.name, localPath);
        }
      }
      catch (err) {
        // Skip unreadable/invalid yaml files during indexing
      }
    }
  }

  entryExists(name: string): boolean {
    const entryPath = this._index.get(name);
    return !!entryPath && fs.existsSync(entryPath);
  }

  listEntries(): string[] {
    return Array.from(this._index.keys());
  }

  async loadEntry(name: string): Promise<md.Entry> {
    const entryPath = this._index.get(name);
    if (!entryPath || !fs.existsSync(entryPath)) {
      throw new Error(`Entry not found: ${name}`);
    }

    const content = await fs.promises.readFile(entryPath, 'utf8');
    return yaml.parse(content) as md.Entry;
  }

  async saveEntry(name: string, entry: md.Entry): Promise<void> {
    const entryPath = path.join(this._catalogPath, `${name}.yaml`);
    await fs.promises.mkdir(path.dirname(entryPath), { recursive: true });

    await fs.promises.writeFile(entryPath, yaml.stringify(entry), 'utf8');
    this._index.set(name, entryPath);
  }

  async deleteEntry(name: string): Promise<void> {
    const entryPath = this._index.get(name);
    if (!entryPath || !fs.existsSync(entryPath)) {
      throw new Error(`Entry not found: ${name}`);
    }

    await fs.promises.unlink(entryPath);
    this._index.delete(name);
  }
}


================================================
FILE: toolbox\mdcode\src\libts\manifest.ts
================================================
// Implements support for creating and loading catalog manifests.
//

import * as fs from 'node:fs';
import * as yaml from 'yaml';
import * as z from 'zod';
import * as gcp from './gcp';
import { CatalogSource, createSource, Sources } from './source';


const manifestSchema = z.object({
  scope: z.union([z.string(), z.array(z.string())]),
  snapshot: z.object({
    entries: z.array(z.string()).optional(),
    aspects: z.array(z.string()).optional()
  }).optional(),
  publishing: z.object({
    entries: z.array(z.string()).optional(),
    aspects: z.array(z.string()).optional()
  }).optional(),
});

export interface SnapshotConfig {
  entries?: string[];
  aspects?: string[];
}

export interface PublishingConfig {
  entries?: string[];
  aspects?: string[];
}

export interface Scope {
  type: string;
  name: string;
}


export class CatalogManifest {
  readonly source: CatalogSource;
  readonly snapshotConfig?: SnapshotConfig;
  readonly publishingConfig?: PublishingConfig;

  private constructor(
    source: CatalogSource,
    snapshotConfig?: SnapshotConfig,
    publishingConfig?: PublishingConfig
  ) {
    this.source = source;
    this.snapshotConfig = snapshotConfig;
    this.publishingConfig = publishingConfig;
  }

  static async initWithEntryGroup(name: string, ctx: gcp.ApiContext): Promise<CatalogManifest> {
    const source = await createSource(Sources.ENTRYGROUP, name, ctx);
    return new CatalogManifest(source);
  }

  static async initWithBigQuery(dataset: string, ctx: gcp.ApiContext): Promise<CatalogManifest> {
    const source = await createSource(Sources.BIGQUERY_DATASET, dataset, ctx);
    return new CatalogManifest(source);
  }

  static async initWithKnowledgeBase(name: string, ctx: gcp.ApiContext): Promise<CatalogManifest> {
    const source = await createSource(Sources.KB, name, ctx);
    return new CatalogManifest(source);
  }

  static async load(path: string, ctx: gcp.ApiContext): Promise<CatalogManifest> {
    const content = fs.readFileSync(path, 'utf8');
    const parsed = yaml.parse(content);
    
    const result = manifestSchema.safeParse(parsed);
    if (!result.success) {
      throw new Error(`Manifest error: ${result.error.message}`);
    }
    
    const scope = result.data.scope;
    let source: CatalogSource;
    if (Array.isArray(scope)) {
      if (scope.length === 0) {
        throw new Error('Manifest error: scope array cannot be empty.');
      }

      const datasets: string[] = [];
      for (const s of scope) {
        const dotIndex = s.indexOf('.');
        if (dotIndex === -1) {
          throw new Error(`Manifest error: scope '${s}' is invalid.`);
        }
        const type = s.substring(0, dotIndex);
        const name = s.substring(dotIndex + 1);
        if (type !== Sources.BIGQUERY_DATASET) {
          throw new Error(`Manifest error: Unsupported scope type in multiple scopes: '${type}'.`);
        }
        datasets.push(name);
      }

      source = await createSource(Sources.BIGQUERY_DATASET, datasets.join(','), ctx);
    }
    else {
      const dotIndex = scope.indexOf('.');
      if (dotIndex === -1) {
        throw new Error(`Manifest error: scope '${scope}' is invalid.`);
      }
      source = await createSource(
        scope.substring(0, dotIndex),
        scope.substring(dotIndex + 1),
        ctx
      );
    }

    const snapshot = result.data.snapshot;
    if (snapshot) {
      if (snapshot.entries) {
        for (const entryType of snapshot.entries) {
          const parts = entryType.split('.');
          if (parts.length !== 3) {
            throw new Error(`Manifest error: Invalid Entry Type '${entryType}'`);
          }
        }
      }

      if (snapshot.aspects) {
        for (const aspectType of snapshot.aspects) {
          const parts = aspectType.split('.');
          if (parts.length !== 3) {
            throw new Error(`Manifest error: Invalid Aspect Type '${aspectType}'`);
          }
        }
      }
    }

    const publishing = result.data.publishing;
    if (publishing) {
      if (publishing.entries) {
        for (const entryType of publishing.entries) {
          const parts = entryType.split('.');
          if (parts.length !== 3) {
            throw new Error(`Manifest error: Invalid Entry Type '${entryType}'`);
          }
          if (!snapshot?.entries?.includes(entryType)) {
            throw new Error(
              `Manifest error: Publishing entry type '${entryType}' is not listed in snapshot entries.`
            );
          }
        }
      }

      if (publishing.aspects) {
        for (const aspectType of publishing.aspects) {
          const parts = aspectType.split('.');
          if (parts.length !== 3) {
            throw new Error(`Manifest error: Invalid Aspect Type '${aspectType}'`);
          }
          if (!snapshot?.aspects?.includes(aspectType)) {
            throw new Error(
              `Manifest error: Publishing aspect type '${aspectType}' is not listed in snapshot aspects.`
            );
          }
        }
      }
    }

    return new CatalogManifest(source, snapshot, publishing);
  }

  save(path: string): void {
    let scope: string | string[];
    const names = this.source.name.split(',');
    if (names.length > 1) {
      scope = names.map(n => `${this.source.type}.${n}`);
    }
    else {
      scope = `${this.source.type}.${this.source.name}`;
    }

    const data: any = {
      scope: scope,
      snapshot: this.snapshotConfig ?? undefined,
      publishing: this.publishingConfig ?? undefined
    };
    fs.writeFileSync(path, yaml.stringify(data), 'utf8');
  }
}


================================================
FILE: toolbox\mdcode\src\libts\metadata.ts
================================================
// Defines metadata objects provided by the catalog snapshot
//

export interface Aspect {
  [key: string]: any;
}

export interface Entry {
  name: string;
  type: string;
  resource: {
    name?: string;
    displayName?: string;
    description?: string;
    labels?: Record<string, string>;
    location?: string;
    parent?: string;
    ancestors?: {
      name: string;
      type: string;
    }[];
    createTime?: string;
    updateTime?: string;
  };
  aspects?: Record<string, Aspect>;
}


================================================
FILE: toolbox\mdcode\src\libts\snapshot.ts
================================================
// Implements a local catalog interface
//

import * as fs from 'node:fs';
import * as path from 'node:path';

import * as gcp from './gcp/context';
import * as dataplex from './gcp/dataplex';
import * as md from './metadata';
import { CatalogManifest } from './manifest';
import { CatalogLayout, createLayout } from './layout';


export class CatalogSnapshot {

  public readonly manifest: CatalogManifest;
  public readonly basePath: string;

  private readonly _entryTypes: Map<string, dataplex.EntryType> = new Map();
  private readonly _aspectTypes: Map<string, dataplex.AspectType> = new Map();

  private readonly _layout: CatalogLayout;

  private constructor(basePath: string, manifest: CatalogManifest) {
    this.basePath = basePath;
    this.manifest = manifest;

    const catalogPath = path.join(this.basePath, 'catalog');
    this._layout = createLayout(manifest.source.layout, catalogPath);
  }

  static async fromPath(basePath: string, ctx: gcp.ApiContext): Promise<CatalogSnapshot> {
    const manifestPath = path.join(basePath, 'catalog.yaml');
    if (!fs.existsSync(manifestPath)) {
      throw new Error(`Cannot find catalog manifest at '${manifestPath}'`);
    }

    const manifest = await CatalogManifest.load(manifestPath, ctx);
    const snapshot = new CatalogSnapshot(basePath, manifest);

    await snapshot._buildTypes(manifest, ctx);
    await snapshot._layout.init();
    return snapshot;
  }

  get entryTypes(): Map<string, dataplex.EntryType> {
    return this._entryTypes;
  }

  get aspectTypes(): Map<string, dataplex.AspectType> {
    return this._aspectTypes;
  }

  // Retrieves the list of locally (pulled and/or created) managed entries
  async listEntries(): Promise<string[]> {
    return this._layout.listEntries();
  }

  // Retrieves the local copy of the entry using its local name
  async lookupEntry(name: string): Promise<md.Entry> {
    return await this._layout.loadEntry(name);
  }

  // Updates the locally managed entry, referenced by its local name.
  // The list of fields can either be "resource" to update the resource-level metadata
  // (which is relevant in case of non-ingested entries) or an aspect identified by it
  // key (project.location.type).
  async updateEntry(entry: md.Entry, fields: string[]): Promise<void> {
    const existingEntry = await this._layout.loadEntry(entry.name);

    for (const f of fields) {
      if (f == 'resource') {
        if (!existingEntry.resource) {
          existingEntry.resource = {};
        }
        if (!entry.resource) {
          entry.resource = {};
        }
        existingEntry.resource.description = entry.resource.description;
      }
      else {
        const aspectType = dataplex._typeRefToName(f, 'aspect');
        if (!this._aspectTypes.has(aspectType)) {
          throw new Error(`The aspect '${f}' is not registered in the snapshot.`);
        }

        if (this.manifest.source.ingestedEntries) {
          const entryType = this._entryTypes.get(existingEntry.type);
          if (!entryType || entryType.requiredAspects?.find((a) => a.type == aspectType)) {
            throw new Error(`The aspect '${f}' is not modifiable on the entry.`);
          }
        }

        if (!existingEntry.aspects) {
          existingEntry.aspects = {};
        }
        if (entry.aspects && entry.aspects[f]) {
          existingEntry.aspects[f] = entry.aspects[f];
        }
        else {
          delete existingEntry.aspects[f];
        }
      }
    }

    await this._layout.saveEntry(entry.name, existingEntry);
  }

  // Creates an entry within the locally catalog snapshot. This capabilitiy is only supported
  // when the associated EntryGroup is user-managed, i.e. not contain ingested metadata.
  async createEntry(name: string, entry: md.Entry): Promise<void> {
    if (this.manifest.source.ingestedEntries) {
      throw new Error(`Entry cannot be created as entries are ingested.`);
    }

    // TODO: Validate aspect and other things

    if (this._layout.entryExists(name)) {
       throw new Error(`Entry '${name}' already exists`);
    }

    await this._layout.saveEntry(name, entry);
  }

  // Deletes an entry within the locally catalog snapshot. This capabilitiy is only supported
  // when the associated EntryGroup is user-managed, i.e. not contain ingested metadata.
  async deleteEntry(name: string): Promise<void> {
    if (this.manifest.source.ingestedEntries) {
      throw new Error(`Entry cannot be deleted as entries are ingested.`);
    }

    await this._layout.deleteEntry(name);
  }

  // Build the map of types supported within the locally managed catalog snapshot
  // Types are stored using two keys: the resource name and the 3-part type name.
  private async _buildTypes(manifest: CatalogManifest, ctx: gcp.ApiContext): Promise<void> {
    const catalog = new dataplex.CatalogClient(ctx);

    for (const entryType of manifest.snapshotConfig?.entries || []) {
      const parts = entryType.split('.');
      const res = await catalog.getEntryType(parts[0], parts[1], parts[2]);
      if (!res.result) {
        throw new Error(`Unable to load type information for entry type ${entryType}`);
      }

      this._entryTypes.set(res.result.name, res.result);
      this._entryTypes.set(entryType, res.result);

      for (const requiredAspect of res.result.requiredAspects ?? []) {
        if (!this._aspectTypes.has(requiredAspect.type)) {
          const parts = requiredAspect.type.split('/');
          const res = await catalog.getAspectType(parts[1], parts[3], parts[5]);
          if (!res.result) {
            throw new Error(`Unable to load type information for aspect type ${requiredAspect.type}`);
          }
          this._aspectTypes.set(res.result.name, res.result);
          this._aspectTypes.set(`${parts[0]}.${parts[3]}.${parts[5]}`, res.result);
        }
      }
    }

    for (const aspectType of manifest.snapshotConfig?.aspects || []) {
      if (this._aspectTypes.has(aspectType)) {
        continue;
      }

      const parts = aspectType.split('.');
      const res = await catalog.getAspectType(parts[0], parts[1], parts[2]);
      if (!res.result) {
        throw new Error(`Unable to load type information for aspect type ${aspectType}`);
      }
      this._aspectTypes.set(res.result.name, res.result);
      this._aspectTypes.set(aspectType, res.result);
    }
  }

  // Stores a Dataplex entry into the locally managed catalog snapshot. This will internally map
  // The service representation into the local metadata representation.
  // This is only meant to be used within the syncing process (as part of pull operations).
  async _storeEntry(entry: dataplex.Entry): Promise<void> {
    const localName = this.manifest.source.localName(entry);
    await this._layout.saveEntry(localName, toLocalEntry(entry, localName));
  }

  // Fetches a Dataplex entry from its local metadata representation.
  // This is only meant to be used within the syncing process (as part of push operations).
  async _fetchEntry(name: string): Promise<dataplex.Entry | undefined> {
    const entry = await this._layout.loadEntry(name);

    if (this.manifest.publishingConfig?.entries?.length &&
        !this.manifest.publishingConfig.entries.includes(entry.type)) {
      return undefined;
    }

    const serviceName = this.manifest.source.serviceName(name);
    if (entry.resource?.parent && !entry.resource.parent.startsWith('projects/')) {
      entry.resource.parent = this.manifest.source.serviceName(entry.resource.parent);
    }
    return toServiceEntry(
      entry,
      serviceName,
      this.manifest,
      this._entryTypes,
      this._aspectTypes
    );
  }
}

// Converts a Dataplex entry into the local metadata representation.
function toLocalEntry(entry: dataplex.Entry, localName: string): md.Entry {
  const aspects: Record<string, md.Aspect> = {};
  if (entry.aspects) {
    for (const key in entry.aspects) {
      aspects[key] = entry.aspects[key].data ?? {};
    }
  }

  const entrySource = entry.entrySource ?? {};

  return {
      name: localName,
      type: dataplex._nameToTypeRef(entry.entryType),
      resource: {
        name: entrySource.resource ?? undefined,
        displayName: entrySource.displayName ?? undefined,
        description: entrySource.description ?? undefined,
        labels: entrySource.labels ?? undefined,
        location: entrySource.location ?? undefined,
        ancestors: entrySource.ancestors ?? undefined,
        createTime: entrySource.createTime ?? undefined,
        updateTime: entrySource.updateTime ?? undefined
      },
      aspects: aspects ?? undefined
  };
}


// Converts a local metadata representation into a Dataplex Entry
function toServiceEntry(entry: md.Entry,
                        serviceName: string,
                        manifest: CatalogManifest,
                        entryTypes: Map<string, dataplex.EntryType>,
                        aspectTypes: Map<string, dataplex.AspectType>): dataplex.Entry {
  const entryType = entryTypes.get(entry.type);
  if (!entryType) {
    throw new Error(`Unknown entry type ${entry.type} in snapshot`);
  }

  const aspects: Record<string, dataplex.Aspect> = {};
  if (entry.aspects) {
    for (const key in entry.aspects) {
      if (manifest.publishingConfig && !manifest.publishingConfig.aspects?.includes(key)) {
        continue;
      }

      const aspectType = dataplex._typeRefToName(key, 'aspect');
      if (manifest.source.ingestedEntries &&
          entryType.requiredAspects?.find((aspectInfo) => aspectInfo.type == aspectType)) {
        continue;
      }

      aspects[key] = { aspectType, data: entry.aspects[key] };
    }
  }

  const resource = entry.resource ?? {};
  const entryTypeName = dataplex._typeRefToName(entry.type, 'entry');

  if (manifest.source.ingestedEntries ||
      !entry.resource || !Object.keys(entry.resource).length) {
    return {
      name: serviceName,
      entryType: entryTypeName,
      aspects: aspects
    };
  }

  return {
    name: serviceName,
    entryType: entryTypeName,
    parentEntry: resource.parent,
    entrySource: {
      resource: resource.name,
      ancestors: resource.ancestors,
      displayName: resource.displayName,
      description: resource.description,
      labels: resource.labels,
      location: resource.location,
      createTime: resource.createTime,
      updateTime: resource.updateTime
    },
    aspects: aspects
  };
}


================================================
FILE: toolbox\mdcode\src\libts\source.ts
================================================
// Defines a Catalog metadata source abstraction
//

import * as gcp from './gcp';
import * as bq from './gcp/bigquery';
import * as dataplex from './gcp/dataplex';
import { Layouts } from './layout';
import { EntryGroupSource } from './sources/entrygroup';
import { BigQueryDatasetSource } from './sources/bq-dataset';
import { KnowledgeBaseSource } from './sources/kb';

export enum Sources {
  ENTRYGROUP = 'entryGroup',
  BIGQUERY_DATASET = 'bq-dataset',
  KB = 'kb'
}


export interface CatalogSource {
  readonly type: string;
  readonly name: string;
  readonly ingestedEntries: boolean;
  readonly layout: Layouts;

  entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown>;
  localName(entry: gcp.Entry): string;
  serviceName(localName: string): string;
}


async function getEntryGroup(name: string, ctx: gcp.ApiContext): Promise<dataplex.EntryGroup> {
  const [project, location, entryGroup] = name.split('.')
  if (!project || !location || !entryGroup) {
    throw new Error('EntryGroup must be in format <projectId>.<locationId>.<entryGroupId>');
  }

  const catalog = new gcp.CatalogClient(ctx);
  const res = await catalog.getEntryGroup(project, location, entryGroup);
  if (!res.result) {
    throw new Error(`Failed to locate EntryGroup '${name}'.`);
  }

  return res.result;
}


async function getBigQueryDatasets(name: string, ctx: gcp.ApiContext): Promise<Map<string, bq.Dataset>> {
  const datasets = new Map<string, bq.Dataset>();
  const names = name.split(',');

  const bigQuery = new bq.BigQueryClient(ctx);
  for (const n of names) {
    const [project, dataset] = n.split('.');
    if (!project || !dataset) {
      throw new Error(`BigQuery dataset must be in format <projectId>.<datasetId>: ${n}`);
    }

    const res = await bigQuery.getDataset(project, dataset);
    if (!res.result) {
      throw new Error(`Failed to locate BigQuery dataset '${n}'.`);
    }

    datasets.set(n, res.result);
  }

  return datasets;
}


export async function createSource(type: string, name: string,
                                   ctx: gcp.ApiContext): Promise<CatalogSource> {
  switch (type) {
    case Sources.ENTRYGROUP:
      const entryGroup = await getEntryGroup(name, ctx);
      return new EntryGroupSource(Sources.ENTRYGROUP, name, entryGroup);
    case Sources.BIGQUERY_DATASET:
      const datasets = await getBigQueryDatasets(name, ctx);
      return new BigQueryDatasetSource(Sources.BIGQUERY_DATASET, name, datasets);
    case Sources.KB:
      const knowledgeBase = await getEntryGroup(name, ctx);
      return new KnowledgeBaseSource(Sources.KB, name, knowledgeBase);
    default:
      throw new Error(`Unknown source type: ${type}`);
  }
}


================================================
FILE: toolbox\mdcode\src\libts\sources\bq-dataset.ts
================================================
// BigQuery Dataset as Metadata Source
//

import * as gcp from '../gcp';
import * as bq from '../gcp/bigquery';
import { Layouts } from '../layout';
import { CatalogSource } from '../source';


export class BigQueryDatasetSource implements CatalogSource {
  readonly type: string;
  readonly name: string;
  readonly ingestedEntries = true;
  readonly layout = Layouts.STANDARD;

  private readonly _datasets: Map<string, bq.Dataset>;

  constructor(type: string, name: string, datasets: Map<string, bq.Dataset>) {
    this.type = type;
    this.name = name;

    this._datasets = datasets;
  }

  async *entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown> {
    const bigQuery = new bq.BigQueryClient(ctx);
    const catalog = new gcp.CatalogClient(ctx);

    for (const [_, dsResource] of this._datasets.entries()) {
      const project = dsResource.datasetReference.projectId;
      const dataset = dsResource.datasetReference.datasetId;

      // Fetch the dataset entry
      const location = dsResource.location.toLowerCase();
      const dsEntryId = `bigquery.googleapis.com/projects/${project}/datasets/${dataset}`
      const dsEntryName = `${gcp.catalogContainer(project, location, '@bigquery')}/entries/${dsEntryId}`
      const dsEntryResult = await catalog.lookupEntry(project, location, dsEntryName);
      if (!dsEntryResult.result) {
        throw new Error(`Failed to get Entry for dataset ${project}.${dataset}`);
      }
      yield dsEntryResult.result;

      // Fetch the table entries
      for await (const table of bigQuery.listTables(project, dataset)) {
        const tableId = table.tableReference.tableId;
        const tableEntryName = `${dsEntryName}/tables/${tableId}`
        const tableEntryResult = await catalog.lookupEntry(project, location, tableEntryName);
        if (!tableEntryResult.result) {
          throw new Error(`Failed to get Entry for table ${project}.${dataset}.${tableId}`);
        }

        yield tableEntryResult.result;
      }
    }

    // TODO: Add support for routines, models
  }

  localName(entry: gcp.Entry): string {
    // The local catalog uses simplified path scheme:
    // dataset -> <project id>.<dataset id>
    // table -> <project id>.<dataset id>/<table id>
    // model -> <project id>.<dataset id>/models/<model id>
    // routine -> <project id>.<dataset id>/routines/<routine id>

    let match = entry.name.match(/\/projects\/([^/]+)\/datasets\/([^/]+)\/(tables|models|routines)\/(.+)$/);
    if (match) {
      const [, project, dataset, type, id] = match;
      if (type === 'tables') {
        return `${project}.${dataset}/${id}`;
      }
      return `${type}/${project}.${dataset}/${id}`;
    }

    match = entry.name.match(/\/projects\/([^/]+)\/datasets\/([^/]+)$/);
    if (match) {
      const [, project, dataset] = match;
      return `${project}.${dataset}`;
    }

    throw new Error(`Invalid BigQuery entry: ${entry.name}`);
  }

  serviceName(localName: string): string {
    const nameParts = localName.split('/');
    const dsResource = this._datasets.get(nameParts[0]);
    if (!dsResource) {
      throw new Error(`Failed to find dataset for ${nameParts[0]}`);
    }

    const project = dsResource.datasetReference.projectId;
    const location = dsResource.location.toLowerCase();
    const dataset = dsResource.datasetReference.datasetId;

    const entryGroup = `${gcp.catalogContainer(project, location, '@bigquery')}`;
    const entryName = `${entryGroup}/entries/bigquery.googleapis.com/projects/${project}/datasets/${dataset}`;
    if (nameParts.length == 1) {
      return entryName;
    }

    const collection = nameParts.length == 2 ? 'tables' : nameParts[1];
    const resource = nameParts.length == 2 ? nameParts[1] : nameParts[2];
    return `${entryName}/${collection}/${resource}`;
  }
}


================================================
FILE: toolbox\mdcode\src\libts\sources\entrygroup.ts
================================================
// Dataplex EntryGroup as Metadata Source
//

import * as gcp from '../gcp';
import * as dataplex from '../gcp/dataplex';
import { Layouts } from '../layout';
import { CatalogSource } from '../source';


export class EntryGroupSource implements CatalogSource {
  readonly type: string;
  readonly name: string;
  readonly ingestedEntries: boolean;
  readonly layout = Layouts.STANDARD;

  private readonly _name: string[];
  private readonly _entryGroup: dataplex.EntryGroup;

  constructor(type: string, name: string, entryGroup: dataplex.EntryGroup) {
    this.type = type;
    this.name = name;

    this._name = name.split('.');
    this._entryGroup = entryGroup;

    this.ingestedEntries = this._name[2].startsWith('@');
  }

  async *entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown> {
    // Enumerate all entries in the EntryGroup

    const catalog = new gcp.CatalogClient(ctx);
    for await (const entry of catalog.listEntries(this._name[0], this._name[1], this._name[2])) {
      yield entry;
    }
  }

  localName(entry: gcp.Entry): string {
    // The local catalog uses the entry id as is
    const match = entry.name.match(/entryGroups\/([^/]+)\/entries\/(.+)$/);
    if (!match) {
      throw new Error(`Invalid entry name for entry: ${entry.name}`);
    }

    return match[2];
  }

  serviceName(localName: string): string {
    return `${this._entryGroup.name}/entries/${localName}`;
  }
}


================================================
FILE: toolbox\mdcode\src\libts\sources\kb.ts
================================================
// Dataplex Wiki managed as an EntryGroup as Metadata Source
//

import * as gcp from '../gcp';
import * as dataplex from '../gcp/dataplex';
import { Layouts } from '../layout';
import { CatalogSource } from '../source';


export class KnowledgeBaseSource implements CatalogSource {
  readonly type: string;
  readonly name: string;
  readonly ingestedEntries = false;
  readonly layout = Layouts.DOCUMENTS;

  private readonly _name: string[];
  private readonly _entryGroup: dataplex.EntryGroup;

  constructor(type: string, name: string, entryGroup: dataplex.EntryGroup) {
    this.type = type;
    this.name = name;

    this._name = name.split('.');
    this._entryGroup = entryGroup;
  }

  async *entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown> {
    // Enumerate all entries in the EntryGroup

    const catalog = new gcp.CatalogClient(ctx);
    for await (const entry of catalog.listEntries(this._name[0], this._name[1], this._name[2])) {
      yield entry;
    }
  }

  localName(entry: gcp.Entry): string {
    // The local catalog uses the entry id as is
    const match = entry.name.match(/entryGroups\/([^/]+)\/entries\/(.+)$/);
    if (!match) {
      throw new Error(`Invalid entry name for entry: ${entry.name}`);
    }

    return match[2];
  }

  serviceName(localName: string): string {
    return `${this._entryGroup.name}/entries/${localName}`;
  }
}


================================================
FILE: toolbox\mdcode\src\libts\sync.ts
================================================
// Implements catalog sync logic for pull and push operations
//

import * as gcp from './gcp';
import { CatalogSnapshot } from './snapshot';

export interface SyncResult {
  success: boolean;
  details?: string;
}

export interface ValidationResult {
  valid: boolean;
}

export interface StatusResult {
  modified: boolean;
}


export class CatalogSync {
  private _catalog: gcp.CatalogClient;
  private _snapshot: CatalogSnapshot;

  constructor(catalog: gcp.CatalogClient, snapshot: CatalogSnapshot) {
    this._catalog = catalog;
    this._snapshot = snapshot;
  }

  // Lists metadata in the Catalog service to create or update the local snapshot.
  async pull(): Promise<SyncResult> {
    try {
      const entries = this._snapshot.manifest.source.entries(this._catalog.context);
      
      for await (const entry of entries) {
        if (this._snapshot.entryTypes.size && !this._snapshot.entryTypes.has(entry.entryType)) {
          continue;
        }

        // TODO: Need to populate type info if its a type we haven't seen.
        // TODO: Handle local modification conflicts.
        // TODO: Handle config changes or service deletions that require removing local entries.

        const nameParts = entry.name.split('/');
        const res = await this._catalog.lookupEntry(nameParts[1], nameParts[3], entry.name,
                                                    [...this._snapshot.aspectTypes.keys()]);
        // The server will respond with 403 permission denied for both resource not exist or 
        // insufficient permission. We cannot tell if a resource not exist or user does not 
        // have the access. Thus using 200 for an ensured result.
        if (res.status != 200 || !res.result) {
          continue;
        }

        await this._snapshot._storeEntry(res.result);
      }
      return { success: true };
    }
    catch (e: any) {
      return { success: false, details: e.message };
    }
  }

  // Pushes local metadata to the Catalog service to publish/deploy it.
  async push(options?: { force?: boolean, validateOnly?: boolean; }): Promise<SyncResult> {
    const entries = await this._snapshot.listEntries();

    for (const name of entries) {
      const entry = await this._snapshot._fetchEntry(name);
      if (!entry) {
        // If this was filtered out based on publishing config
        continue;
      }

      // TODO: Track what has changed and do minimal update.
      // TODO: Handle creates and deletes, as well as partial updates.
      // TODO: Handle conflicts.

      const nameParts = entry.name.split('/');
      const project = nameParts[1];
      const location = nameParts[3];

      const exist = await this._catalog.lookupEntry(project, location, entry.name);
      if (exist.status != 200 || !exist.result) {
        const entryGroup = nameParts[5];
        const entryId = nameParts.slice(7).join('/');
        const createEntryRes = await this._catalog.createEntry(project, location, entryGroup, entryId, entry);
        if (createEntryRes.status != 200 || !createEntryRes.result) {
          return { success: false, details: `Failed to create entry ${entry.name}: ${createEntryRes.message || createEntryRes.status}` };
        }
        continue;
      }

      const updateMask = [];
      const aspectKeys = Object.keys(entry.aspects || {});
      if (aspectKeys.length) {
        updateMask.push('aspects');
      }

      if (!this._snapshot.manifest.source.ingestedEntries) {
        if (entry.entrySource) {
          updateMask.push('entry_source');
        }
        if (entry.parentEntry) {
          updateMask.push('parent_entry');
        }
      }

      if (!updateMask.length) {
        continue;
      }

      const res = await this._catalog.modifyEntry(project, location, entry, updateMask, aspectKeys);
      if (res.status !== 200 || !res.result) {
        return { success: false, details: `Failed to update entry ${name}: ${res.message || res.status}` };
      }
    }

    return { success: true };
  }

  async validate(): Promise<ValidationResult> {
    throw new Error('Not yet implemented');
  }

  async status(): Promise<StatusResult> {
    throw new Error('Not yet implemented');
  }
}


================================================
FILE: toolbox\mdcode\src\tool\commands.ts
================================================
// CLI command handlers
//

import * as fs from 'node:fs';

import * as kcmd from '../libts';
import * as dataplex from '../libts/gcp/dataplex';
import * as context from '../libts/gcp/context';


export interface InitOptions {
  entryGroup?: string;
  bigqueryDataset?: string | string[];
  kb?: string;
  pull?: boolean;
}


export interface PushOptions {
  force?: boolean;
  validateOnly?: boolean;
}


export async function init(options: InitOptions): Promise<number> {
  const ctx = context.ApiContext.default();

  let manifest: kcmd.CatalogManifest;
  if (options.entryGroup) {
    manifest = await kcmd.CatalogManifest.initWithEntryGroup(options.entryGroup, ctx);
  }
  else if (options.kb) {
    manifest = await kcmd.CatalogManifest.initWithKnowledgeBase(options.kb, ctx);
  }
  else if (options.bigqueryDataset) {
    let datasets = '';
    if (Array.isArray(options.bigqueryDataset)) {
      datasets = options.bigqueryDataset.join(',');
    }
    else {
      datasets = options.bigqueryDataset!;
    }
    manifest = await kcmd.CatalogManifest.initWithBigQuery(datasets, ctx);
  }
  else {
    console.error('Error: Must provide either --entry-group or --bigquery-dataset or --kb');
    return 1;
  }

  manifest.save('catalog.yaml');
  console.log(fs.readFileSync('catalog.yaml', 'utf8'));

  if (options.pull) {
    return await pull();
  }

  return 0;
}


export async function pull(): Promise<number> {
  const ctx = context.ApiContext.default();
  const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx);

  const catalog = new dataplex.CatalogClient(ctx);
  const sync = new kcmd.CatalogSync(catalog, snapshot);

  console.log('Pulling catalog entries...');
  const result = await sync.pull();

  if (result.success) {
    console.log('Successfully updated local snapshot.');
    return 0;
  }
  else {
    console.error('Error pulling catalog entries:', result.details);
    return 1;
  }
}


export async function push(options: PushOptions): Promise<number> {
  const ctx = context.ApiContext.default();
  const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx);

  const catalog = new dataplex.CatalogClient(ctx);
  const sync = new kcmd.CatalogSync(catalog, snapshot);

  console.log('Pushing catalog entries...');
  const result = await sync.push(options);

  if (result.success) {
    console.log('Successfully pushed catalog entries.');
    return 0;
  }
  else {
    console.error('Error pushing catalog entries:', result.details);
    return 1;
  }
}


================================================
FILE: toolbox\mdcode\src\tool\main.ts
================================================
// Main CLI entrypoint
//

import * as cac from 'cac';
import * as commands from './commands';
import * as mcp from './mcp';


const cli = cac.cac('kcmd').version('1.0.0').help();
cli.command('init', 'Initialize a new catalog snapshot')
   .option('--entry-group <id>', 'Identifier of the EntryGroup (project.location.id)')
   .option('--bigquery-dataset <id...>', 'Identifier of the BigQuery dataset(s) (project.datasetId)')
   .option('--kb <id>', 'Identifier of the Knowledge Base EntryGroup (project.location.id)')
   .option('--pull', 'Optionally pull catalog entries during initialization')
   .action(async (options) => {
      try {
        await commands.init(options);
      }
      catch (err: any) {
        console.error('Error:', err.message || err);
        process.exit(1);
      }
   });


cli.command('pull', 'Pull catalog entries')
   .action(async () => {
      let exitCode = 1;
      try {
        exitCode = await commands.pull();
      }
      catch (err: any) {
        console.error('Error:', err.message || err);
        exitCode = 1;
      }
      
      process.exit(exitCode);
   });

cli.command('push', 'Push catalog entries')
   .option('--force', 'Force push changes')
   .option('--validate-only', 'Only validate changes without applying')
   .action(async (options) => {
      let exitCode = 1;
      try {
        exitCode = await commands.push(options);
      }
      catch (err: any) {
        console.error('Error:', err.message || err);
        exitCode = 1;
      }
      
      process.exit(exitCode);
   });


cli.command('mcp', 'Run the Model Context Protocol (MCP) server')
   .option('--path <path>', 'Path to the catalog snapshot root directory')
   .action(async (options) => {
      try {
        await mcp.startServer(options.path);
      }
      catch (err: any) {
        console.error('Error starting MCP server:', err.message || err);
        process.exit(1);
      }
   });


cli.parse();

if (!cli.matchedCommand) {
  if (cli.args.length > 0) {
    console.error(`Error: Unknown command '${cli.args[0]}'`);
  }

  cli.outputHelp();
  process.exit(1);
}


================================================
FILE: toolbox\mdcode\src\tool\mcp.ts
================================================
// MCP Server implementation
//

import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { z } from 'zod';

import * as kcmd from '../libts';
import * as gcp from '../libts/gcp';

export async function startServer(basePath: string = '.') {
  const server = new McpServer({
    name: 'kcmd',
    version: '1.0.0',
  });

  const ctx = gcp.ApiContext.default();
  const snapshot = await kcmd.CatalogSnapshot.fromPath(basePath, ctx);

  server.registerTool(
    'list-entries',
    {
      description: 'List names of all catalog entries. ' +
        'Each entry corresponds to a resource that has associated metadata.',
    },
    async () => {
      const names = await snapshot.listEntries();
      return {
        content: [{ type: 'text', text: JSON.stringify(names, null, 2) }],
      };
    }
  );

  server.registerTool(
    'lookup-entry',
    {
      description: 'Lookup the metadata of a specific catalog entry by name',
      inputSchema: {
        name: z.string().describe('The name of the entry to lookup'),
      },
    },
    async ({ name }) => {
      try {
        const entry = await snapshot.lookupEntry(name);
        return {
          content: [{ type: 'text', text: JSON.stringify(entry, null, 2) }],
        };
      }
      catch (error: any) {
        return {
          isError: true,
          content: [{ type: 'text', text: `Error looking up entry: ${error.message}` }],
        };
      }
    }
  );

  server.registerTool(
    'modify-entry',
    {
      description: 'Modify (either "resource" or an aspect key field) of a catalog entry by name',
      inputSchema: {
        name: z.string().describe('The name of the entry to modify'),
        field: z.string().describe('The name of the field being updated. Either "resource" or an aspect key'),
        updates: z.record(z.string(), z.any()).describe('A structured JSON data dictionary containing the updates'),
      },
    },
    async ({ name, field, updates }) => {
      try {
        const existingEntry = await snapshot.lookupEntry(name);
        let updatedEntry: kcmd.Entry = {
          name,
          type: existingEntry.type,
          resource: field === 'resource' ? updates : {},
          aspects: field !== 'resource' ? { [field]: updates } : undefined,
        };

        await snapshot.updateEntry(updatedEntry, [field]);

        updatedEntry = await snapshot.lookupEntry(name);
        return {
          content: [{ type: 'text', text: JSON.stringify(updatedEntry, null, 2) }],
        };
      }
      catch (error: any) {
        return {
          isError: true,
          content: [{ type: 'text', text: `Error modifying entry: ${error.message}` }],
        };
      }
    }
  );

  const transport = new StdioServerTransport();
  await server.connect(transport);
}


================================================
FILE: toolbox\mdcode\tests\libts\mocks.ts
================================================
import * as gcp from '../../src/libts/gcp';
import * as bigquery from '../../src/libts/gcp/bigquery';

// Bypass actual gcloud CLI calls by using the explicit constructor
export const TEST_API_CONTEXT = new gcp.ApiContext('test-project', 'test-location', 'test-token');


export class CatalogClientMock extends gcp.CatalogClient {
  public mockEntries: gcp.Entry[] = [];
  public mockEntryGroups: Map<string, gcp.EntryGroup> = new Map();
  public mockEntryTypes: Map<string, gcp.EntryType> = new Map();
  public mockAspectTypes: Map<string, gcp.AspectType> = new Map();

  constructor() {
    super(TEST_API_CONTEXT);
  }

  setMockEntries(entries: gcp.Entry[]) {
    this.mockEntries = entries;
  }

  addMockEntryGroup(resource: gcp.EntryGroup) {
    this.mockEntryGroups.set(resource.name, resource);
  }

  addMockEntryType(resource: gcp.EntryType) {
    this.mockEntryTypes.set(resource.name, resource);
  }

  addMockAspectType(resource: gcp.AspectType) {
    this.mockAspectTypes.set(resource.name, resource);
  }

  async getEntryGroup(project: string, location: string, id: string): Promise<gcp.ApiResult<gcp.EntryGroup>> {
    const name = `projects/${project}/locations/${location}/entryGroups/${id}`;
    const group = this.mockEntryGroups.get(name);
    if (group) {
      return { status: 200, result: group };
    }
    return { status: 404, message: 'Not found' };
  }

  async getEntryType(project: string, location: string, id: string): Promise<gcp.ApiResult<gcp.EntryType>> {
    const name = `projects/${project}/locations/${location}/entryTypes/${id}`;
    const res = this.mockEntryTypes.get(name);
    if (res) {
      return { status: 200, result: res };
    }
    return { status: 404, message: 'Not found' };
  }

  async getAspectType(project: string, location: string, id: string): Promise<gcp.ApiResult<gcp.AspectType>> {
    const name = `projects/${project}/locations/${location}/aspectTypes/${id}`;
    const res = this.mockAspectTypes.get(name);
    if (res) {
      return { status: 200, result: res };
    }
    return { status: 404, message: 'Not found' };
  }

  async getEntry(project: string, location: string, entryGroup: string, id: string,
                 aspects?: string[]): Promise<gcp.ApiResult<gcp.Entry>> {
    const name = `projects/${project}/locations/${location}/entryGroups/${entryGroup}/entries/${id}`;
    const entry = this.mockEntries.find(e => e.name == name);
    if (entry) {
      return { status: 200, result: entry };
    }
    return { status: 404, message: 'Not found' };
  }

  async lookupEntry(project: string, location: string, name: string, aspects?: string[]): Promise<gcp.ApiResult<gcp.Entry>> {
    const entry = this.mockEntries.find(e => e.name == name);
    if (entry) {
      return { status: 200, result: entry };
    }
    return { status: 404, message: 'Not found' };
  }

  async modifyEntry(project: string, location: string, entry: gcp.Entry, updateMask?: string[], aspectKeys?: string[]): Promise<gcp.ApiResult<gcp.Entry>> {
    const existingEntry = this.mockEntries.find(e => e.name == entry.name);
    if (existingEntry) {
      if (updateMask?.find(m => m == 'entry_source')) {
        existingEntry.entrySource = entry.entrySource;
      }
      if (updateMask?.find(m => m == 'aspects')) {
        if (!existingEntry.aspects) {
          existingEntry.aspects = {};
        }
        for (const aspectKey of aspectKeys ?? []) {
          if (entry.aspects?.[aspectKey]) {
            existingEntry.aspects[aspectKey] = entry.aspects[aspectKey];
          }
          else {
            delete existingEntry.aspects[aspectKey];
          }
        }
      }
      return { status: 200, result: existingEntry };
    }
    return { status: 404, message: 'Not found' };
  }

  async *listEntries(project: string, location: string,
                     entryGroup: string): AsyncGenerator<gcp.Entry, void, unknown> {
    for (const entry of this.mockEntries) {
      yield entry;
    }
  }

  async updateEntry(entry: gcp.Entry, updateMask?: string[], aspectKeys?: string[]): Promise<gcp.ApiResult<gcp.Entry>> {
    const existingEntry = this.mockEntries.find(e => e.name == entry.name);
    if (existingEntry) {
      if (updateMask?.find(m => m == 'entry_source')) {
        existingEntry.entrySource = entry.entrySource;
      }
      if (updateMask?.find(m => m == 'aspects')) {
        if (!existingEntry.aspects) {
          existingEntry.aspects = {};
        }
        for (const f in aspectKeys ?? []) {
          if (entry.aspects?.[f]) {
            existingEntry.aspects[f] = entry.aspects[f];
          }
          else {
            delete existingEntry.aspects[f];
          }
        }
      }
      return { status: 200, result: existingEntry };
    }
    return { status: 404, message: 'Not found' };
  }

  async createEntry(project: string, location: string, entryGroup: string, entryId: string, entry?: gcp.Entry): Promise<gcp.ApiResult<gcp.Entry>> {
    const fakeEntry = entry;
    if (fakeEntry) {
      this.mockEntries.push(fakeEntry);
      return { status: 200, result: entry };
    }
    return {status: 404, message: 'Not found' };
  }
}


export class BigQueryClientMock extends bigquery.BigQueryClient {
  public mockDatasets: Map<string, any> = new Map();
  public mockTables: Map<string, any> = new Map();

  constructor() {
    super(TEST_API_CONTEXT);
  }

  addMockDataset(resource: bigquery.Dataset) {
    const name = `projects/${resource.datasetReference.projectId}/datasets/${resource.datasetReference.datasetId}`;
    this.mockDatasets.set(name, resource);
  }

  addMockTable(resource: bigquery.Table) {
    const name = `projects/${resource.tableReference.projectId}/datasets/${resource.tableReference.datasetId}/tables/${resource.tableReference.tableId}`;
    this.mockTables.set(name, resource);
  }

  async getDataset(project: string, id: string): Promise<gcp.ApiResult<bigquery.Dataset>> {
    const name = `projects/${project}/datasets/${id}`;
    const resource = this.mockDatasets.get(name);
    if (resource) {
      return { status: 200, result: resource };
    }
    return { status: 404, message: 'Not found' };
  }

  async *listTables(project: string, dataset: string): AsyncGenerator<bigquery.Table> {
    for (const table of this.mockTables.values()) {
      if (table.tableReference.projectId === project && table.tableReference.datasetId === dataset) {
        yield table;
      }
    }
  }
}


================================================
FILE: toolbox\mdcode\tests\libts\scenarios.ts
================================================
// Library test runner
//

import * as fs from 'node:fs';
import * as glob from 'glob';
import * as path from 'node:path';
import * as yaml from 'yaml';
import { describe, test, expect, mock, spyOn } from 'bun:test';
import { fs as memfs, vol } from 'memfs';
import * as kcmac from '../../src/libts';
import * as gcp from '../../src/libts/gcp';
import * as bq from '../../src/libts/gcp/bigquery';

import { CatalogClientMock, BigQueryClientMock, TEST_API_CONTEXT } from './mocks';

let currentCatalogMock: CatalogClientMock | null = null;
let currentBigQueryMock: BigQueryClientMock | null = null;


function runScenario(scenario: any) {
  describe(scenario.name, () => {
    test('run', async () => {
      const catalog = new CatalogClientMock();
      const bigquery = new BigQueryClientMock();

      // Reset state
      vol.reset();
      currentCatalogMock = catalog;
      currentBigQueryMock = bigquery;

      // Setup state - Catalog Service
      if (scenario.setup?.catalog?.entries) {
        catalog.setMockEntries(scenario.setup.catalog.entries);
      }
      if (scenario.setup?.catalog?.entryGroups) {
        for (const eg of scenario.setup.catalog.entryGroups) {
          catalog.addMockEntryGroup(eg);
        }
      }
      if (scenario.setup?.catalog?.entryTypes) {
        for (const et of scenario.setup.catalog.entryTypes) {
          catalog.addMockEntryType(et);
        }
      }
      if (scenario.setup?.catalog?.aspectTypes) {
        for (const at of scenario.setup.catalog.aspectTypes) {
          catalog.addMockAspectType(at);
        }
      }

      // Setup state - BigQuery Service
      if (scenario.setup?.bigQuery?.datasets) {
        for (const ds of scenario.setup.bigQuery.datasets) {
          bigquery.addMockDataset(ds);
        }
      }
      if (scenario.setup?.bigQuery?.tables) {
        for (const table of scenario.setup.bigQuery.tables) {
          bigquery.addMockTable(table);
        }
      }

      // Setup state - Filesystem
      if (scenario.setup?.fileSystem) {
        vol.fromJSON(scenario.setup.fileSystem, '/');
      }
      else {
        vol.fromJSON({}, '/');
      }

      // Execute - Manifest setup
      if (scenario.init?.entryGroup) {
        const mf = await kcmac.CatalogManifest.initWithEntryGroup(
          scenario.init.entryGroup, TEST_API_CONTEXT);
        mf.save('/catalog.yaml');
      }
      if (scenario.init?.dataset) {
        const mf = await kcmac.CatalogManifest.initWithBigQuery(
          scenario.init.dataset, TEST_API_CONTEXT);
        mf.save('/catalog.yaml');
      }
      if (scenario.init?.kb) {
        const mf = await kcmac.CatalogManifest.initWithKnowledgeBase(
          scenario.init.kb, TEST_API_CONTEXT);
        mf.save('/catalog.yaml');
      }

      // Execute - Snapshot creation
      if (!fs.existsSync('/catalog.yaml')) {
        throw new Error('Scenario did not include or initialize a manifest');
      }
      const snapshot = await kcmac.CatalogSnapshot.fromPath('/', TEST_API_CONTEXT);
      const sync = new kcmac.CatalogSync(catalog, snapshot);

      // Execute - Snapshot actions
      const actions = scenario.actions ?? [];
      for (const actionStep of actions) {
        const { action, ...params } = actionStep;
        switch (action) {
          case 'pull':
            await sync.pull();
            break;
          case 'push':
            await sync.push(params.options);
            break;
          case 'listEntries':
            console.log(await snapshot.listEntries());
            break;
          case 'createEntry':
            await snapshot.createEntry(params.name, params.entry);
            break;
          case 'updateEntry':
            await snapshot.updateEntry(params.entry, params.fields);
            break;
          case 'deleteEntry':
            await snapshot.deleteEntry(params.name);
            break;
          default:
            throw new Error(`Unknown action: ${action}`);
        }
      }

      // Assert expectations - Filesystem
      if (scenario.assert?.fileSystem) {
        for (const [fpath, rawCondition] of Object.entries(scenario.assert.fileSystem)) {
          const condition = rawCondition as any;
          const absolutePath = path.resolve('/', fpath);

          if (condition === null) {
            expect(fs.existsSync(absolutePath)).toBe(false);
          }
          else {
            expect(fs.existsSync(absolutePath)).toBe(true);
            if (typeof condition === 'string') {
              const actualContent = fs.readFileSync(absolutePath, 'utf8') as string;
              expect(actualContent.trim()).toBe(condition.trim());
            }
            else if (condition && typeof condition === 'object' && 'contains' in condition) {
              const actualContent = fs.readFileSync(absolutePath, 'utf8') as string;
              expect(actualContent).toContain(condition.contains);
            }
          }
        }
      }

      // Assert expectations - Catalog Service
      if (scenario.assert?.catalog?.entries) {
        expect(JSON.parse(JSON.stringify(catalog.mockEntries))).toEqual(JSON.parse(JSON.stringify(scenario.assert.catalog.entries)));
      }
    });
  });
}

function main() {
  let globPattern = '**/*.yaml';
  if (process.env.TEST_GLOB) {
    globPattern = process.env.TEST_GLOB;
  }

  const scenariosPath = path.join(__dirname, '..', 'scenarios');
  const files = glob.globSync(globPattern, { cwd: scenariosPath, absolute: true });
  const scenarios = files.map(file => yaml.parse(fs.readFileSync(file, 'utf-8')));

  // Establish dynamic prototype spies to automatically connect inner-constructed 
  // API clients directly to the scenario mock data registries.
  spyOn(gcp.CatalogClient.prototype, 'getEntryGroup').mockImplementation(
    async function(project: string, location: string, id: string) {
      if (currentCatalogMock) {
        return await currentCatalogMock.getEntryGroup(project, location, id);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'getEntryType').mockImplementation(
    async function(project: string, location: string, type: string) {
      if (currentCatalogMock) {
        return await currentCatalogMock.getEntryType(project, location, type);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'getAspectType').mockImplementation(
    async function(project: string, location: string, type: string) {
      if (currentCatalogMock) {
        return await currentCatalogMock.getAspectType(project, location, type);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'getEntry').mockImplementation(
    async function(project: string, location: string, entryGroup: string, entry: string) {
      if (currentCatalogMock) {
        return await currentCatalogMock.getEntry(project, location, entryGroup, entry);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'lookupEntry').mockImplementation(
    async function(project: string, location: string, entry: string) {
      if (currentCatalogMock) {
        return await currentCatalogMock.lookupEntry(project, location, entry);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'modifyEntry').mockImplementation(
    async function(project: string, location: string, entry: gcp.Entry, updateMask?: string[], aspectKeys?: string[]) {
      if (currentCatalogMock) {
        return await currentCatalogMock.modifyEntry(project, location, entry, updateMask, aspectKeys);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'listEntries').mockImplementation(
    async function* (project: string, location: string, entryGroup: string) {
      if (currentCatalogMock) {
        for await (const entry of currentCatalogMock.listEntries(project, location, entryGroup)) {
          yield entry;
        }
      }
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'updateEntry').mockImplementation(
    async function(entry: gcp.Entry, updateMask?: string[], aspectKeys?: string[]) {
      if (currentCatalogMock) {
        return await currentCatalogMock.updateEntry(entry, updateMask, aspectKeys);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(gcp.CatalogClient.prototype, 'createEntry').mockImplementation(
    async function(project: string, location: string, entryGroup: string, entryId: string, entry?: gcp.Entry) {
      if (currentCatalogMock) {
        return await currentCatalogMock.createEntry(project, location, entryGroup, entryId, entry);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(bq.BigQueryClient.prototype, 'getDataset').mockImplementation(
    async function(project: string, dataset: string) {
      if (currentBigQueryMock) {
        return await currentBigQueryMock.getDataset(project, dataset);
      }
      return { status: 404, message: 'Not found' };
    }
  );

  spyOn(bq.BigQueryClient.prototype, 'listTables').mockImplementation(
    async function* (project: string, dataset: string) {
      if (currentBigQueryMock) {
        for await (const table of currentBigQueryMock.listTables(project, dataset)) {
          yield table;
        }
      }
    }
  );

  // Globally mock node:fs to direct file system calls to virtual volume
  mock.module('node:fs', () => memfs);

  for (const scenario of scenarios) {
    runScenario(scenario);
  }
}

main();


================================================
FILE: toolbox\README.md
================================================
# Toolbox

Contains tools to work with Knowledge Catalog metadata.

* [Metadata as Code](./mdcode/README.md)
  Provides the ability to manage metadata in the form of source code artifacts that
  can be sync'd with metadata in Knowledge Catalog.

* [Enrichment Agent](./enrichment/README.md)
  Provides an ready-to-use agent and customizable harness to produce, evolve/improve
  and maintain metadata within Knowledge Catalog and make it ready for consumption by
  agents.
