Import tensorflow

This commit is contained in:
derek committed 2026-02-15 21:45:42 -08:00
1 parent f3e8b90764
commit c530630153
20524 files changed
+9017694 -25

No files matched your search

@@ -0,0 +1,113 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorBoard is a webapp for understanding TensorFlow runs and graphs."""
from tensorboard import lazy as _lazy
from tensorboard import version as _version
# TensorBoard public API.
__all__ = [
"__version__",
"errors",
"notebook",
"program",
"summary",
]
# Please be careful when changing the structure of this file.
#
# The lazy imports in this file must use `importlib.import_module`, not
# `import tensorboard.foo` or `from tensorboard import foo`, or it will
# be impossible to reload the TensorBoard module without breaking these
# top-level public APIs. This has to do with the gory details of
# Python's module system. Take `tensorboard.notebook` as an example:
#
# - When the `tensorboard` module (that's us!) is initialized, its
# `notebook` attribute is initialized to a new LazyModule. The
# actual `tensorboard.notebook` submodule is not loaded.
#
# - When the `tensorboard.notebook` submodule is first loaded, Python
# _reassigns_ the `notebook` attribute on the `tensorboard` module
# object to point to the underlying `tensorboard.notebook` module
# object, rather than its former LazyModule value. This occurs
# whether the module is loaded via the lazy module or directly as an
# import:
#
# - import tensorboard; tensorboard.notebook.start(...) # one way
# - from tensorboard import notebook # other way; same effect
#
# - When the `tensorboard` module is reloaded, its `notebook`
# attribute is once again bound to a (new) LazyModule, while the
# `tensorboard.notebook` module object is unaffected and still
# exists in `sys.modules`. But then...
#
# - When the new LazyModule is forced, it must resolve to the existing
# `tensorboard.notebook` module object rather than itself (which
# just creates a stack overflow). If the LazyModule load function
# uses `import tensorboard.notebook; return tensorboard.notebook`,
# then the first statement will do _nothing_ because the
# `tensorboard.notebook` module is already loaded, and the second
# statement will return the LazyModule itself. The same goes for the
# `from tensorboard import notebook` form. We need to ensure that
# the submodule is loaded and then pull the actual module object out
# of `sys.modules`... which is exactly what `importlib` handles for
# us.
#
# See <https://github.com/tensorflow/tensorboard/issues/1989> for
# additional discussion.
@_lazy.lazy_load("tensorboard.errors")
def errors():
import importlib
return importlib.import_module("tensorboard.errors")
@_lazy.lazy_load("tensorboard.notebook")
def notebook():
import importlib
return importlib.import_module("tensorboard.notebook")
@_lazy.lazy_load("tensorboard.program")
def program():
import importlib
return importlib.import_module("tensorboard.program")
@_lazy.lazy_load("tensorboard.summary")
def summary():
import importlib
return importlib.import_module("tensorboard.summary")
def load_ipython_extension(ipython):
"""IPython API entry point.
Only intended to be called by the IPython runtime.
See:
https://ipython.readthedocs.io/en/stable/config/extensions/index.html
"""
notebook._load_ipython_extension(ipython)
__version__ = _version.VERSION
@@ -0,0 +1,125 @@
from tensorboard._vendor.bleach.linkifier import (
DEFAULT_CALLBACKS,
Linker,
)
from tensorboard._vendor.bleach.sanitizer import (
ALLOWED_ATTRIBUTES,
ALLOWED_PROTOCOLS,
ALLOWED_TAGS,
Cleaner,
)
# yyyymmdd
__releasedate__ = "20241029"
# x.y.z or x.y.z.dev0 -- semver
__version__ = "6.2.0"
__all__ = ["clean", "linkify"]
def clean(
text,
tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
protocols=ALLOWED_PROTOCOLS,
strip=False,
strip_comments=True,
css_sanitizer=None,
):
"""Clean an HTML fragment of malicious content and return it
This function is a security-focused function whose sole purpose is to
remove malicious content from a string such that it can be displayed as
content in a web page.
This function is not designed to use to transform content to be used in
non-web-page contexts.
Example::
import bleach
better_text = bleach.clean(yucky_text)
.. Note::
If you're cleaning a lot of text and passing the same argument values or
you want more configurability, consider using a
:py:class:`bleach.sanitizer.Cleaner` instance.
:arg str text: the text to clean
:arg set tags: set of allowed tags; defaults to
``bleach.sanitizer.ALLOWED_TAGS``
:arg dict attributes: allowed attributes; can be a callable, list or dict;
defaults to ``bleach.sanitizer.ALLOWED_ATTRIBUTES``
:arg list protocols: allowed list of protocols for links; defaults
to ``bleach.sanitizer.ALLOWED_PROTOCOLS``
:arg bool strip: whether or not to strip disallowed elements
:arg bool strip_comments: whether or not to strip HTML comments
:arg CSSSanitizer css_sanitizer: instance with a "sanitize_css" method for
sanitizing style attribute values and style text; defaults to None
:returns: cleaned text as unicode
"""
cleaner = Cleaner(
tags=tags,
attributes=attributes,
protocols=protocols,
strip=strip,
strip_comments=strip_comments,
css_sanitizer=css_sanitizer,
)
return cleaner.clean(text)
def linkify(text, callbacks=DEFAULT_CALLBACKS, skip_tags=None, parse_email=False):
"""Convert URL-like strings in an HTML fragment to links
This function converts strings that look like URLs, domain names and email
addresses in text that may be an HTML fragment to links, while preserving:
1. links already in the string
2. urls found in attributes
3. email addresses
linkify does a best-effort approach and tries to recover from bad
situations due to crazy text.
.. Note::
If you're linking a lot of text and passing the same argument values or
you want more configurability, consider using a
:py:class:`bleach.linkifier.Linker` instance.
.. Note::
If you have text that you want to clean and then linkify, consider using
the :py:class:`bleach.linkifier.LinkifyFilter` as a filter in the clean
pass. That way you're not parsing the HTML twice.
:arg str text: the text to linkify
:arg list callbacks: list of callbacks to run when adjusting tag attributes;
defaults to ``bleach.linkifier.DEFAULT_CALLBACKS``
:arg list skip_tags: list of tags that you don't want to linkify the
contents of; for example, you could set this to ``['pre']`` to skip
linkifying contents of ``pre`` tags
:arg bool parse_email: whether or not to linkify email addresses
:returns: linkified text as unicode
"""
linker = Linker(callbacks=callbacks, skip_tags=skip_tags, parse_email=parse_email)
return linker.linkify(text)
@@ -0,0 +1,35 @@
"""
HTML parsing library based on the `WHATWG HTML specification
<https://whatwg.org/html>`_. The parser is designed to be compatible with
existing HTML found in the wild and implements well-defined error recovery that
is largely compatible with modern desktop web browsers.
Example usage::
import html5lib
with open("my_document.html", "rb") as f:
tree = html5lib.parse(f)
For convenience, this module re-exports the following names:
* :func:`~.html5parser.parse`
* :func:`~.html5parser.parseFragment`
* :class:`~.html5parser.HTMLParser`
* :func:`~.treebuilders.getTreeBuilder`
* :func:`~.treewalkers.getTreeWalker`
* :func:`~.serializer.serialize`
"""
from __future__ import absolute_import, division, unicode_literals
from .html5parser import HTMLParser, parse, parseFragment
from .treebuilders import getTreeBuilder
from .treewalkers import getTreeWalker
from .serializer import serialize
__all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
"getTreeWalker", "serialize"]
# this has to be at the top level, see how setup.py parses this
#: Distribution version number.
__version__ = "1.1"
@@ -0,0 +1,289 @@
from __future__ import absolute_import, division, unicode_literals
import re
import warnings
from .constants import DataLossWarning
baseChar = """
[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] |
[#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] |
[#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] |
[#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 |
[#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] |
[#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] |
[#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] |
[#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] |
[#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 |
[#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] |
[#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] |
[#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D |
[#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] |
[#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] |
[#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] |
[#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] |
[#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] |
[#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] |
[#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 |
[#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] |
[#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] |
[#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] |
[#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] |
[#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] |
[#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] |
[#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] |
[#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] |
[#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] |
[#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] |
[#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A |
#x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 |
#x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] |
#x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] |
[#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] |
[#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C |
#x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 |
[#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] |
[#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] |
[#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 |
[#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] |
[#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B |
#x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE |
[#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] |
[#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 |
[#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] |
[#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]"""
ideographic = """[#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]"""
combiningCharacter = """
[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] |
[#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 |
[#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] |
[#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] |
#x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] |
[#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] |
[#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 |
#x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] |
[#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC |
[#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] |
#x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] |
[#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] |
[#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] |
[#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] |
[#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] |
[#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] |
#x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 |
[#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] |
#x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] |
[#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] |
[#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] |
#x3099 | #x309A"""
digit = """
[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] |
[#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] |
[#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] |
[#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]"""
extender = """
#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 |
#[#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]"""
letter = " | ".join([baseChar, ideographic])
# Without the
name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter,
extender])
nameFirst = " | ".join([letter, "_"])
reChar = re.compile(r"#x([\d|A-F]{4,4})")
reCharRange = re.compile(r"\[#x([\d|A-F]{4,4})-#x([\d|A-F]{4,4})\]")
def charStringToList(chars):
charRanges = [item.strip() for item in chars.split(" | ")]
rv = []
for item in charRanges:
foundMatch = False
for regexp in (reChar, reCharRange):
match = regexp.match(item)
if match is not None:
rv.append([hexToInt(item) for item in match.groups()])
if len(rv[-1]) == 1:
rv[-1] = rv[-1] * 2
foundMatch = True
break
if not foundMatch:
assert len(item) == 1
rv.append([ord(item)] * 2)
rv = normaliseCharList(rv)
return rv
def normaliseCharList(charList):
charList = sorted(charList)
for item in charList:
assert item[1] >= item[0]
rv = []
i = 0
while i < len(charList):
j = 1
rv.append(charList[i])
while i + j < len(charList) and charList[i + j][0] <= rv[-1][1] + 1:
rv[-1][1] = charList[i + j][1]
j += 1
i += j
return rv
# We don't really support characters above the BMP :(
max_unicode = int("FFFF", 16)
def missingRanges(charList):
rv = []
if charList[0] != 0:
rv.append([0, charList[0][0] - 1])
for i, item in enumerate(charList[:-1]):
rv.append([item[1] + 1, charList[i + 1][0] - 1])
if charList[-1][1] != max_unicode:
rv.append([charList[-1][1] + 1, max_unicode])
return rv
def listToRegexpStr(charList):
rv = []
for item in charList:
if item[0] == item[1]:
rv.append(escapeRegexp(chr(item[0])))
else:
rv.append(escapeRegexp(chr(item[0])) + "-" +
escapeRegexp(chr(item[1])))
return "[%s]" % "".join(rv)
def hexToInt(hex_str):
return int(hex_str, 16)
def escapeRegexp(string):
specialCharacters = (".", "^", "$", "*", "+", "?", "{", "}",
"[", "]", "|", "(", ")", "-")
for char in specialCharacters:
string = string.replace(char, "\\" + char)
return string
# output from the above
nonXmlNameBMPRegexp = re.compile('[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') # noqa
nonXmlNameFirstBMPRegexp = re.compile('[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') # noqa
# Simpler things
nonPubidCharRegexp = re.compile("[^\x20\x0D\x0Aa-zA-Z0-9\\-'()+,./:=?;!*#@$_%]")
class InfosetFilter(object):
replacementRegexp = re.compile(r"U[\dA-F]{5,5}")
def __init__(self,
dropXmlnsLocalName=False,
dropXmlnsAttrNs=False,
preventDoubleDashComments=False,
preventDashAtCommentEnd=False,
replaceFormFeedCharacters=True,
preventSingleQuotePubid=False):
self.dropXmlnsLocalName = dropXmlnsLocalName
self.dropXmlnsAttrNs = dropXmlnsAttrNs
self.preventDoubleDashComments = preventDoubleDashComments
self.preventDashAtCommentEnd = preventDashAtCommentEnd
self.replaceFormFeedCharacters = replaceFormFeedCharacters
self.preventSingleQuotePubid = preventSingleQuotePubid
self.replaceCache = {}
def coerceAttribute(self, name, namespace=None):
if self.dropXmlnsLocalName and name.startswith("xmlns:"):
warnings.warn("Attributes cannot begin with xmlns", DataLossWarning)
return None
elif (self.dropXmlnsAttrNs and
namespace == "http://www.w3.org/2000/xmlns/"):
warnings.warn("Attributes cannot be in the xml namespace", DataLossWarning)
return None
else:
return self.toXmlName(name)
def coerceElement(self, name):
return self.toXmlName(name)
def coerceComment(self, data):
if self.preventDoubleDashComments:
while "--" in data:
warnings.warn("Comments cannot contain adjacent dashes", DataLossWarning)
data = data.replace("--", "- -")
if data.endswith("-"):
warnings.warn("Comments cannot end in a dash", DataLossWarning)
data += " "
return data
def coerceCharacters(self, data):
if self.replaceFormFeedCharacters:
for _ in range(data.count("\x0C")):
warnings.warn("Text cannot contain U+000C", DataLossWarning)
data = data.replace("\x0C", " ")
# Other non-xml characters
return data
def coercePubid(self, data):
dataOutput = data
for char in nonPubidCharRegexp.findall(data):
warnings.warn("Coercing non-XML pubid", DataLossWarning)
replacement = self.getReplacementCharacter(char)
dataOutput = dataOutput.replace(char, replacement)
if self.preventSingleQuotePubid and dataOutput.find("'") >= 0:
warnings.warn("Pubid cannot contain single quote", DataLossWarning)
dataOutput = dataOutput.replace("'", self.getReplacementCharacter("'"))
return dataOutput
def toXmlName(self, name):
nameFirst = name[0]
nameRest = name[1:]
m = nonXmlNameFirstBMPRegexp.match(nameFirst)
if m:
warnings.warn("Coercing non-XML name: %s" % name, DataLossWarning)
nameFirstOutput = self.getReplacementCharacter(nameFirst)
else:
nameFirstOutput = nameFirst
nameRestOutput = nameRest
replaceChars = set(nonXmlNameBMPRegexp.findall(nameRest))
for char in replaceChars:
warnings.warn("Coercing non-XML name: %s" % name, DataLossWarning)
replacement = self.getReplacementCharacter(char)
nameRestOutput = nameRestOutput.replace(char, replacement)
return nameFirstOutput + nameRestOutput
def getReplacementCharacter(self, char):
if char in self.replaceCache:
replacement = self.replaceCache[char]
else:
replacement = self.escapeChar(char)
return replacement
def fromXmlName(self, name):
for item in set(self.replacementRegexp.findall(name)):
name = name.replace(item, self.unescapeChar(item))
return name
def escapeChar(self, char):
replacement = "U%05X" % ord(char)
self.replaceCache[char] = replacement
return replacement
def unescapeChar(self, charcode):
return chr(int(charcode[1:], 16))
@@ -0,0 +1,918 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
from tensorboard._vendor.bleach.six_shim import http_client, urllib
import codecs
import re
from io import BytesIO, StringIO
from tensorboard._vendor import webencodings
from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
from .constants import _ReparseException
from . import _utils
# Non-unicode versions of constants for use in the pre-parser
spaceCharactersBytes = frozenset([item.encode("ascii") for item in spaceCharacters])
asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]" # noqa
if _utils.supports_lone_surrogates:
# Use one extra step of indirection and create surrogates with
# eval. Not using this indirection would introduce an illegal
# unicode literal on platforms not supporting such lone
# surrogates.
assert invalid_unicode_no_surrogate[-1] == "]" and invalid_unicode_no_surrogate.count("]") == 1
invalid_unicode_re = re.compile(invalid_unicode_no_surrogate[:-1] +
eval('"\\uD800-\\uDFFF"') + # pylint:disable=eval-used
"]")
else:
invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
non_bmp_invalid_codepoints = {0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
0x10FFFE, 0x10FFFF}
ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005C\u005B-\u0060\u007B-\u007E]")
# Cache for charsUntil()
charsUntilRegEx = {}
class BufferedStream(object):
"""Buffering for streams that do not have buffering of their own
The buffer is implemented as a list of chunks on the assumption that
joining many strings will be slow since it is O(n**2)
"""
def __init__(self, stream):
self.stream = stream
self.buffer = []
self.position = [-1, 0] # chunk number, offset
def tell(self):
pos = 0
for chunk in self.buffer[:self.position[0]]:
pos += len(chunk)
pos += self.position[1]
return pos
def seek(self, pos):
assert pos <= self._bufferedBytes()
offset = pos
i = 0
while len(self.buffer[i]) < offset:
offset -= len(self.buffer[i])
i += 1
self.position = [i, offset]
def read(self, bytes):
if not self.buffer:
return self._readStream(bytes)
elif (self.position[0] == len(self.buffer) and
self.position[1] == len(self.buffer[-1])):
return self._readStream(bytes)
else:
return self._readFromBuffer(bytes)
def _bufferedBytes(self):
return sum([len(item) for item in self.buffer])
def _readStream(self, bytes):
data = self.stream.read(bytes)
self.buffer.append(data)
self.position[0] += 1
self.position[1] = len(data)
return data
def _readFromBuffer(self, bytes):
remainingBytes = bytes
rv = []
bufferIndex = self.position[0]
bufferOffset = self.position[1]
while bufferIndex < len(self.buffer) and remainingBytes != 0:
assert remainingBytes > 0
bufferedData = self.buffer[bufferIndex]
if remainingBytes <= len(bufferedData) - bufferOffset:
bytesToRead = remainingBytes
self.position = [bufferIndex, bufferOffset + bytesToRead]
else:
bytesToRead = len(bufferedData) - bufferOffset
self.position = [bufferIndex, len(bufferedData)]
bufferIndex += 1
rv.append(bufferedData[bufferOffset:bufferOffset + bytesToRead])
remainingBytes -= bytesToRead
bufferOffset = 0
if remainingBytes:
rv.append(self._readStream(remainingBytes))
return b"".join(rv)
def HTMLInputStream(source, **kwargs):
# Work around Python bug #20007: read(0) closes the connection.
# http://bugs.python.org/issue20007
if (isinstance(source, http_client.HTTPResponse) or
# Also check for addinfourl wrapping HTTPResponse
(isinstance(source, urllib.response.addbase) and
isinstance(source.fp, http_client.HTTPResponse))):
isUnicode = False
elif hasattr(source, "read"):
isUnicode = isinstance(source.read(0), text_type)
else:
isUnicode = isinstance(source, text_type)
if isUnicode:
encodings = [x for x in kwargs if x.endswith("_encoding")]
if encodings:
raise TypeError("Cannot set an encoding with a unicode input, set %r" % encodings)
return HTMLUnicodeInputStream(source, **kwargs)
else:
return HTMLBinaryInputStream(source, **kwargs)
class HTMLUnicodeInputStream(object):
"""Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
"""
_defaultChunkSize = 10240
def __init__(self, source):
"""Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
"""
if not _utils.supports_lone_surrogates:
# Such platforms will have already checked for such
# surrogate errors, so no need to do this checking.
self.reportCharacterErrors = None
elif len("\U0010FFFF") == 1:
self.reportCharacterErrors = self.characterErrorsUCS4
else:
self.reportCharacterErrors = self.characterErrorsUCS2
# List of where new lines occur
self.newLines = [0]
self.charEncoding = (lookupEncoding("utf-8"), "certain")
self.dataStream = self.openStream(source)
self.reset()
def reset(self):
self.chunk = ""
self.chunkSize = 0
self.chunkOffset = 0
self.errors = []
# number of (complete) lines in previous chunks
self.prevNumLines = 0
# number of columns in the last line of the previous chunk
self.prevNumCols = 0
# Deal with CR LF and surrogates split over chunk boundaries
self._bufferedCharacter = None
def openStream(self, source):
"""Produces a file object from source.
source can be either a file object, local filename or a string.
"""
# Already a file object
if hasattr(source, 'read'):
stream = source
else:
stream = StringIO(source)
return stream
def _position(self, offset):
chunk = self.chunk
nLines = chunk.count('\n', 0, offset)
positionLine = self.prevNumLines + nLines
lastLinePos = chunk.rfind('\n', 0, offset)
if lastLinePos == -1:
positionColumn = self.prevNumCols + offset
else:
positionColumn = offset - (lastLinePos + 1)
return (positionLine, positionColumn)
def position(self):
"""Returns (line, col) of the current position in the stream."""
line, col = self._position(self.chunkOffset)
return (line + 1, col)
def char(self):
""" Read one character from the stream or queue if available. Return
EOF when EOF is reached.
"""
# Read a new chunk from the input stream if necessary
if self.chunkOffset >= self.chunkSize:
if not self.readChunk():
return EOF
chunkOffset = self.chunkOffset
char = self.chunk[chunkOffset]
self.chunkOffset = chunkOffset + 1
return char
def readChunk(self, chunkSize=None):
if chunkSize is None:
chunkSize = self._defaultChunkSize
self.prevNumLines, self.prevNumCols = self._position(self.chunkSize)
self.chunk = ""
self.chunkSize = 0
self.chunkOffset = 0
data = self.dataStream.read(chunkSize)
# Deal with CR LF and surrogates broken across chunks
if self._bufferedCharacter:
data = self._bufferedCharacter + data
self._bufferedCharacter = None
elif not data:
# We have no more data, bye-bye stream
return False
if len(data) > 1:
lastv = ord(data[-1])
if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF:
self._bufferedCharacter = data[-1]
data = data[:-1]
if self.reportCharacterErrors:
self.reportCharacterErrors(data)
# Replace invalid characters
data = data.replace("\r\n", "\n")
data = data.replace("\r", "\n")
self.chunk = data
self.chunkSize = len(data)
return True
def characterErrorsUCS4(self, data):
for _ in range(len(invalid_unicode_re.findall(data))):
self.errors.append("invalid-codepoint")
def characterErrorsUCS2(self, data):
# Someone picked the wrong compile option
# You lose
skip = False
for match in invalid_unicode_re.finditer(data):
if skip:
continue
codepoint = ord(match.group())
pos = match.start()
# Pretty sure there should be endianness issues here
if _utils.isSurrogatePair(data[pos:pos + 2]):
# We have a surrogate pair!
char_val = _utils.surrogatePairToCodepoint(data[pos:pos + 2])
if char_val in non_bmp_invalid_codepoints:
self.errors.append("invalid-codepoint")
skip = True
elif (codepoint >= 0xD800 and codepoint <= 0xDFFF and
pos == len(data) - 1):
self.errors.append("invalid-codepoint")
else:
skip = False
self.errors.append("invalid-codepoint")
def charsUntil(self, characters, opposite=False):
""" Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
"""
# Use a cache of regexps to find the required characters
try:
chars = charsUntilRegEx[(characters, opposite)]
except KeyError:
if __debug__:
for c in characters:
assert(ord(c) < 128)
regex = "".join(["\\x%02x" % ord(c) for c in characters])
if not opposite:
regex = "^%s" % regex
chars = charsUntilRegEx[(characters, opposite)] = re.compile("[%s]+" % regex)
rv = []
while True:
# Find the longest matching prefix
m = chars.match(self.chunk, self.chunkOffset)
if m is None:
# If nothing matched, and it wasn't because we ran out of chunk,
# then stop
if self.chunkOffset != self.chunkSize:
break
else:
end = m.end()
# If not the whole chunk matched, return everything
# up to the part that didn't match
if end != self.chunkSize:
rv.append(self.chunk[self.chunkOffset:end])
self.chunkOffset = end
break
# If the whole remainder of the chunk matched,
# use it all and read the next chunk
rv.append(self.chunk[self.chunkOffset:])
if not self.readChunk():
# Reached EOF
break
r = "".join(rv)
return r
def unget(self, char):
# Only one character is allowed to be ungotten at once - it must
# be consumed again before any further call to unget
if char is not EOF:
if self.chunkOffset == 0:
# unget is called quite rarely, so it's a good idea to do
# more work here if it saves a bit of work in the frequently
# called char and charsUntil.
# So, just prepend the ungotten character onto the current
# chunk:
self.chunk = char + self.chunk
self.chunkSize += 1
else:
self.chunkOffset -= 1
assert self.chunk[self.chunkOffset] == char
class HTMLBinaryInputStream(HTMLUnicodeInputStream):
"""Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
"""
def __init__(self, source, override_encoding=None, transport_encoding=None,
same_origin_parent_encoding=None, likely_encoding=None,
default_encoding="windows-1252", useChardet=True):
"""Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
"""
# Raw Stream - for unicode objects this will encode to utf-8 and set
# self.charEncoding as appropriate
self.rawStream = self.openStream(source)
HTMLUnicodeInputStream.__init__(self, self.rawStream)
# Encoding Information
# Number of bytes to use when looking for a meta element with
# encoding information
self.numBytesMeta = 1024
# Number of bytes to use when using detecting encoding using chardet
self.numBytesChardet = 100
# Things from args
self.override_encoding = override_encoding
self.transport_encoding = transport_encoding
self.same_origin_parent_encoding = same_origin_parent_encoding
self.likely_encoding = likely_encoding
self.default_encoding = default_encoding
# Determine encoding
self.charEncoding = self.determineEncoding(useChardet)
assert self.charEncoding[0] is not None
# Call superclass
self.reset()
def reset(self):
self.dataStream = self.charEncoding[0].codec_info.streamreader(self.rawStream, 'replace')
HTMLUnicodeInputStream.reset(self)
def openStream(self, source):
"""Produces a file object from source.
source can be either a file object, local filename or a string.
"""
# Already a file object
if hasattr(source, 'read'):
stream = source
else:
stream = BytesIO(source)
try:
stream.seek(stream.tell())
except Exception:
stream = BufferedStream(stream)
return stream
def determineEncoding(self, chardet=True):
# BOMs take precedence over everything
# This will also read past the BOM if present
charEncoding = self.detectBOM(), "certain"
if charEncoding[0] is not None:
return charEncoding
# If we've been overridden, we've been overridden
charEncoding = lookupEncoding(self.override_encoding), "certain"
if charEncoding[0] is not None:
return charEncoding
# Now check the transport layer
charEncoding = lookupEncoding(self.transport_encoding), "certain"
if charEncoding[0] is not None:
return charEncoding
# Look for meta elements with encoding information
charEncoding = self.detectEncodingMeta(), "tentative"
if charEncoding[0] is not None:
return charEncoding
# Parent document encoding
charEncoding = lookupEncoding(self.same_origin_parent_encoding), "tentative"
if charEncoding[0] is not None and not charEncoding[0].name.startswith("utf-16"):
return charEncoding
# "likely" encoding
charEncoding = lookupEncoding(self.likely_encoding), "tentative"
if charEncoding[0] is not None:
return charEncoding
# Guess with chardet, if available
if chardet:
try:
from chardet.universaldetector import UniversalDetector
except ImportError:
pass
else:
buffers = []
detector = UniversalDetector()
while not detector.done:
buffer = self.rawStream.read(self.numBytesChardet)
assert isinstance(buffer, bytes)
if not buffer:
break
buffers.append(buffer)
detector.feed(buffer)
detector.close()
encoding = lookupEncoding(detector.result['encoding'])
self.rawStream.seek(0)
if encoding is not None:
return encoding, "tentative"
# Try the default encoding
charEncoding = lookupEncoding(self.default_encoding), "tentative"
if charEncoding[0] is not None:
return charEncoding
# Fallback to html5lib's default if even that hasn't worked
return lookupEncoding("windows-1252"), "tentative"
def changeEncoding(self, newEncoding):
assert self.charEncoding[1] != "certain"
newEncoding = lookupEncoding(newEncoding)
if newEncoding is None:
return
if newEncoding.name in ("utf-16be", "utf-16le"):
newEncoding = lookupEncoding("utf-8")
assert newEncoding is not None
elif newEncoding == self.charEncoding[0]:
self.charEncoding = (self.charEncoding[0], "certain")
else:
self.rawStream.seek(0)
self.charEncoding = (newEncoding, "certain")
self.reset()
raise _ReparseException("Encoding changed from %s to %s" % (self.charEncoding[0], newEncoding))
def detectBOM(self):
"""Attempts to detect at BOM at the start of the stream. If
an encoding can be determined from the BOM return the name of the
encoding otherwise return None"""
bomDict = {
codecs.BOM_UTF8: 'utf-8',
codecs.BOM_UTF16_LE: 'utf-16le', codecs.BOM_UTF16_BE: 'utf-16be',
codecs.BOM_UTF32_LE: 'utf-32le', codecs.BOM_UTF32_BE: 'utf-32be'
}
# Go to beginning of file and read in 4 bytes
string = self.rawStream.read(4)
assert isinstance(string, bytes)
# Try detecting the BOM using bytes from the string
encoding = bomDict.get(string[:3]) # UTF-8
seek = 3
if not encoding:
# Need to detect UTF-32 before UTF-16
encoding = bomDict.get(string) # UTF-32
seek = 4
if not encoding:
encoding = bomDict.get(string[:2]) # UTF-16
seek = 2
# Set the read position past the BOM if one was found, otherwise
# set it to the start of the stream
if encoding:
self.rawStream.seek(seek)
return lookupEncoding(encoding)
else:
self.rawStream.seek(0)
return None
def detectEncodingMeta(self):
"""Report the encoding declared by the meta element
"""
buffer = self.rawStream.read(self.numBytesMeta)
assert isinstance(buffer, bytes)
parser = EncodingParser(buffer)
self.rawStream.seek(0)
encoding = parser.getEncoding()
if encoding is not None and encoding.name in ("utf-16be", "utf-16le"):
encoding = lookupEncoding("utf-8")
return encoding
class EncodingBytes(bytes):
"""String-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raised"""
def __new__(self, value):
assert isinstance(value, bytes)
return bytes.__new__(self, value.lower())
def __init__(self, value):
# pylint:disable=unused-argument
self._position = -1
def __iter__(self):
return self
def __next__(self):
p = self._position = self._position + 1
if p >= len(self):
raise StopIteration
elif p < 0:
raise TypeError
return self[p:p + 1]
def next(self):
# Py2 compat
return self.__next__()
def previous(self):
p = self._position
if p >= len(self):
raise StopIteration
elif p < 0:
raise TypeError
self._position = p = p - 1
return self[p:p + 1]
def setPosition(self, position):
if self._position >= len(self):
raise StopIteration
self._position = position
def getPosition(self):
if self._position >= len(self):
raise StopIteration
if self._position >= 0:
return self._position
else:
return None
position = property(getPosition, setPosition)
def getCurrentByte(self):
return self[self.position:self.position + 1]
currentByte = property(getCurrentByte)
def skip(self, chars=spaceCharactersBytes):
"""Skip past a list of characters"""
p = self.position # use property for the error-checking
while p < len(self):
c = self[p:p + 1]
if c not in chars:
self._position = p
return c
p += 1
self._position = p
return None
def skipUntil(self, chars):
p = self.position
while p < len(self):
c = self[p:p + 1]
if c in chars:
self._position = p
return c
p += 1
self._position = p
return None
def matchBytes(self, bytes):
"""Look for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone"""
rv = self.startswith(bytes, self.position)
if rv:
self.position += len(bytes)
return rv
def jumpTo(self, bytes):
"""Look for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the match"""
try:
self._position = self.index(bytes, self.position) + len(bytes) - 1
except ValueError:
raise StopIteration
return True
class EncodingParser(object):
"""Mini parser for detecting character encoding from meta elements"""
def __init__(self, data):
"""string - the data to work on for encoding detection"""
self.data = EncodingBytes(data)
self.encoding = None
def getEncoding(self):
if b"<meta" not in self.data:
return None
methodDispatch = (
(b"<!--", self.handleComment),
(b"<meta", self.handleMeta),
(b"</", self.handlePossibleEndTag),
(b"<!", self.handleOther),
(b"<?", self.handleOther),
(b"<", self.handlePossibleStartTag))
for _ in self.data:
keepParsing = True
try:
self.data.jumpTo(b"<")
except StopIteration:
break
for key, method in methodDispatch:
if self.data.matchBytes(key):
try:
keepParsing = method()
break
except StopIteration:
keepParsing = False
break
if not keepParsing:
break
return self.encoding
def handleComment(self):
"""Skip over comments"""
return self.data.jumpTo(b"-->")
def handleMeta(self):
if self.data.currentByte not in spaceCharactersBytes:
# if we have <meta not followed by a space so just keep going
return True
# We have a valid meta element we want to search for attributes
hasPragma = False
pendingEncoding = None
while True:
# Try to find the next attribute after the current position
attr = self.getAttribute()
if attr is None:
return True
else:
if attr[0] == b"http-equiv":
hasPragma = attr[1] == b"content-type"
if hasPragma and pendingEncoding is not None:
self.encoding = pendingEncoding
return False
elif attr[0] == b"charset":
tentativeEncoding = attr[1]
codec = lookupEncoding(tentativeEncoding)
if codec is not None:
self.encoding = codec
return False
elif attr[0] == b"content":
contentParser = ContentAttrParser(EncodingBytes(attr[1]))
tentativeEncoding = contentParser.parse()
if tentativeEncoding is not None:
codec = lookupEncoding(tentativeEncoding)
if codec is not None:
if hasPragma:
self.encoding = codec
return False
else:
pendingEncoding = codec
def handlePossibleStartTag(self):
return self.handlePossibleTag(False)
def handlePossibleEndTag(self):
next(self.data)
return self.handlePossibleTag(True)
def handlePossibleTag(self, endTag):
data = self.data
if data.currentByte not in asciiLettersBytes:
# If the next byte is not an ascii letter either ignore this
# fragment (possible start tag case) or treat it according to
# handleOther
if endTag:
data.previous()
self.handleOther()
return True
c = data.skipUntil(spacesAngleBrackets)
if c == b"<":
# return to the first step in the overall "two step" algorithm
# reprocessing the < byte
data.previous()
else:
# Read all attributes
attr = self.getAttribute()
while attr is not None:
attr = self.getAttribute()
return True
def handleOther(self):
return self.data.jumpTo(b">")
def getAttribute(self):
"""Return a name,value pair for the next attribute in the stream,
if one is found, or None"""
data = self.data
# Step 1 (skip chars)
c = data.skip(spaceCharactersBytes | frozenset([b"/"]))
assert c is None or len(c) == 1
# Step 2
if c in (b">", None):
return None
# Step 3
attrName = []
attrValue = []
# Step 4 attribute name
while True:
if c == b"=" and attrName:
break
elif c in spaceCharactersBytes:
# Step 6!
c = data.skip()
break
elif c in (b"/", b">"):
return b"".join(attrName), b""
elif c in asciiUppercaseBytes:
attrName.append(c.lower())
elif c is None:
return None
else:
attrName.append(c)
# Step 5
c = next(data)
# Step 7
if c != b"=":
data.previous()
return b"".join(attrName), b""
# Step 8
next(data)
# Step 9
c = data.skip()
# Step 10
if c in (b"'", b'"'):
# 10.1
quoteChar = c
while True:
# 10.2
c = next(data)
# 10.3
if c == quoteChar:
next(data)
return b"".join(attrName), b"".join(attrValue)
# 10.4
elif c in asciiUppercaseBytes:
attrValue.append(c.lower())
# 10.5
else:
attrValue.append(c)
elif c == b">":
return b"".join(attrName), b""
elif c in asciiUppercaseBytes:
attrValue.append(c.lower())
elif c is None:
return None
else:
attrValue.append(c)
# Step 11
while True:
c = next(data)
if c in spacesAngleBrackets:
return b"".join(attrName), b"".join(attrValue)
elif c in asciiUppercaseBytes:
attrValue.append(c.lower())
elif c is None:
return None
else:
attrValue.append(c)
class ContentAttrParser(object):
def __init__(self, data):
assert isinstance(data, bytes)
self.data = data
def parse(self):
try:
# Check if the attr name is charset
# otherwise return
self.data.jumpTo(b"charset")
self.data.position += 1
self.data.skip()
if not self.data.currentByte == b"=":
# If there is no = sign keep looking for attrs
return None
self.data.position += 1
self.data.skip()
# Look for an encoding between matching quote marks
if self.data.currentByte in (b'"', b"'"):
quoteMark = self.data.currentByte
self.data.position += 1
oldPosition = self.data.position
if self.data.jumpTo(quoteMark):
return self.data[oldPosition:self.data.position]
else:
return None
else:
# Unquoted value
oldPosition = self.data.position
try:
self.data.skipUntil(spaceCharactersBytes)
return self.data[oldPosition:self.data.position]
except StopIteration:
# Return the whole remaining value
return self.data[oldPosition:]
except StopIteration:
return None
def lookupEncoding(encoding):
"""Return the python codec name corresponding to an encoding or None if the
string doesn't correspond to a valid encoding."""
if isinstance(encoding, bytes):
try:
encoding = encoding.decode("ascii")
except UnicodeDecodeError:
return None
if encoding is not None:
try:
return webencodings.lookup(encoding)
except AttributeError:
return None
else:
return None
@@ -0,0 +1,5 @@
from __future__ import absolute_import, division, unicode_literals
from .py import Trie
__all__ = ["Trie"]
@@ -0,0 +1,40 @@
from __future__ import absolute_import, division, unicode_literals
try:
from collections.abc import Mapping
except ImportError: # Python 2.7
from collections import Mapping
class Trie(Mapping):
"""Abstract base class for tries"""
def keys(self, prefix=None):
# pylint:disable=arguments-differ
keys = super(Trie, self).keys()
if prefix is None:
return set(keys)
return {x for x in keys if x.startswith(prefix)}
def has_keys_with_prefix(self, prefix):
for key in self.keys():
if key.startswith(prefix):
return True
return False
def longest_prefix(self, prefix):
if prefix in self:
return prefix
for i in range(1, len(prefix) + 1):
if prefix[:-i] in self:
return prefix[:-i]
raise KeyError(prefix)
def longest_prefix_item(self, prefix):
lprefix = self.longest_prefix(prefix)
return (lprefix, self[lprefix])
@@ -0,0 +1,67 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
from bisect import bisect_left
from ._base import Trie as ABCTrie
class Trie(ABCTrie):
def __init__(self, data):
if not all(isinstance(x, text_type) for x in data.keys()):
raise TypeError("All keys must be strings")
self._data = data
self._keys = sorted(data.keys())
self._cachestr = ""
self._cachepoints = (0, len(data))
def __contains__(self, key):
return key in self._data
def __len__(self):
return len(self._data)
def __iter__(self):
return iter(self._data)
def __getitem__(self, key):
return self._data[key]
def keys(self, prefix=None):
if prefix is None or prefix == "" or not self._keys:
return set(self._keys)
if prefix.startswith(self._cachestr):
lo, hi = self._cachepoints
start = i = bisect_left(self._keys, prefix, lo, hi)
else:
start = i = bisect_left(self._keys, prefix)
keys = set()
if start == len(self._keys):
return keys
while self._keys[i].startswith(prefix):
keys.add(self._keys[i])
i += 1
self._cachestr = prefix
self._cachepoints = (start, i)
return keys
def has_keys_with_prefix(self, prefix):
if prefix in self._data:
return True
if prefix.startswith(self._cachestr):
lo, hi = self._cachepoints
i = bisect_left(self._keys, prefix, lo, hi)
else:
i = bisect_left(self._keys, prefix)
if i == len(self._keys):
return False
return self._keys[i].startswith(prefix)
@@ -0,0 +1,159 @@
from __future__ import absolute_import, division, unicode_literals
from types import ModuleType
try:
from collections.abc import Mapping
except ImportError:
from collections import Mapping
from tensorboard._vendor.bleach.six_shim import text_type, PY3
if PY3:
import xml.etree.ElementTree as default_etree
else:
try:
import xml.etree.cElementTree as default_etree
except ImportError:
import xml.etree.ElementTree as default_etree
__all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
"surrogatePairToCodepoint", "moduleFactoryFactory",
"supports_lone_surrogates"]
# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
# caught by the below test. In general this would be any platform
# using UTF-16 as its encoding of unicode strings, such as
# Jython. This is because UTF-16 itself is based on the use of such
# surrogates, and there is no mechanism to further escape such
# escapes.
try:
_x = eval('"\\uD800"') # pylint:disable=eval-used
if not isinstance(_x, text_type):
# We need this with u"" because of http://bugs.jython.org/issue2039
_x = eval('u"\\uD800"') # pylint:disable=eval-used
assert isinstance(_x, text_type)
except Exception:
supports_lone_surrogates = False
else:
supports_lone_surrogates = True
class MethodDispatcher(dict):
"""Dict with 2 special properties:
On initiation, keys that are lists, sets or tuples are converted to
multiple keys so accessing any one of the items in the original
list-like object returns the matching value
md = MethodDispatcher({("foo", "bar"):"baz"})
md["foo"] == "baz"
A default value which can be set through the default attribute.
"""
def __init__(self, items=()):
_dictEntries = []
for name, value in items:
if isinstance(name, (list, tuple, frozenset, set)):
for item in name:
_dictEntries.append((item, value))
else:
_dictEntries.append((name, value))
dict.__init__(self, _dictEntries)
assert len(self) == len(_dictEntries)
self.default = None
def __getitem__(self, key):
return dict.get(self, key, self.default)
def __get__(self, instance, owner=None):
return BoundMethodDispatcher(instance, self)
class BoundMethodDispatcher(Mapping):
"""Wraps a MethodDispatcher, binding its return values to `instance`"""
def __init__(self, instance, dispatcher):
self.instance = instance
self.dispatcher = dispatcher
def __getitem__(self, key):
# see https://docs.python.org/3/reference/datamodel.html#object.__get__
# on a function, __get__ is used to bind a function to an instance as a bound method
return self.dispatcher[key].__get__(self.instance)
def get(self, key, default):
if key in self.dispatcher:
return self[key]
else:
return default
def __iter__(self):
return iter(self.dispatcher)
def __len__(self):
return len(self.dispatcher)
def __contains__(self, key):
return key in self.dispatcher
# Some utility functions to deal with weirdness around UCS2 vs UCS4
# python builds
def isSurrogatePair(data):
return (len(data) == 2 and
ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and
ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF)
def surrogatePairToCodepoint(data):
char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 +
(ord(data[1]) - 0xDC00))
return char_val
# Module Factory Factory (no, this isn't Java, I know)
# Here to stop this being duplicated all over the place.
def moduleFactoryFactory(factory):
moduleCache = {}
def moduleFactory(baseModule, *args, **kwargs):
if isinstance(ModuleType.__name__, type("")):
name = "_%s_factory" % baseModule.__name__
else:
name = b"_%s_factory" % baseModule.__name__
kwargs_tuple = tuple(kwargs.items())
try:
return moduleCache[name][args][kwargs_tuple]
except KeyError:
mod = ModuleType(name)
objs = factory(baseModule, *args, **kwargs)
mod.__dict__.update(objs)
if "name" not in moduleCache:
moduleCache[name] = {}
if "args" not in moduleCache[name]:
moduleCache[name][args] = {}
if "kwargs" not in moduleCache[name][args]:
moduleCache[name][args][kwargs_tuple] = {}
moduleCache[name][args][kwargs_tuple] = mod
return mod
return moduleFactory
def memoize(func):
cache = {}
def wrapped(*args, **kwargs):
key = (tuple(args), tuple(kwargs.items()))
if key not in cache:
cache[key] = func(*args, **kwargs)
return cache[key]
return wrapped
@@ -0,0 +1,29 @@
from __future__ import absolute_import, division, unicode_literals
from . import base
from collections import OrderedDict
def _attr_key(attr):
"""Return an appropriate key for an attribute for sorting
Attributes have a namespace that can be either ``None`` or a string. We
can't compare the two because they're different types, so we convert
``None`` to an empty string first.
"""
return (attr[0][0] or ''), attr[0][1]
class Filter(base.Filter):
"""Alphabetizes attributes for elements"""
def __iter__(self):
for token in base.Filter.__iter__(self):
if token["type"] in ("StartTag", "EmptyTag"):
attrs = OrderedDict()
for name, value in sorted(token["data"].items(),
key=_attr_key):
attrs[name] = value
token["data"] = attrs
yield token
@@ -0,0 +1,12 @@
from __future__ import absolute_import, division, unicode_literals
class Filter(object):
def __init__(self, source):
self.source = source
def __iter__(self):
return iter(self.source)
def __getattr__(self, name):
return getattr(self.source, name)
@@ -0,0 +1,73 @@
from __future__ import absolute_import, division, unicode_literals
from . import base
class Filter(base.Filter):
"""Injects ``<meta charset=ENCODING>`` tag into head of document"""
def __init__(self, source, encoding):
"""Creates a Filter
:arg source: the source token stream
:arg encoding: the encoding to set
"""
base.Filter.__init__(self, source)
self.encoding = encoding
def __iter__(self):
state = "pre_head"
meta_found = (self.encoding is None)
pending = []
for token in base.Filter.__iter__(self):
type = token["type"]
if type == "StartTag":
if token["name"].lower() == "head":
state = "in_head"
elif type == "EmptyTag":
if token["name"].lower() == "meta":
# replace charset with actual encoding
has_http_equiv_content_type = False
for (namespace, name), value in token["data"].items():
if namespace is not None:
continue
elif name.lower() == 'charset':
token["data"][(namespace, name)] = self.encoding
meta_found = True
break
elif name == 'http-equiv' and value.lower() == 'content-type':
has_http_equiv_content_type = True
else:
if has_http_equiv_content_type and (None, "content") in token["data"]:
token["data"][(None, "content")] = 'text/html; charset=%s' % self.encoding
meta_found = True
elif token["name"].lower() == "head" and not meta_found:
# insert meta into empty head
yield {"type": "StartTag", "name": "head",
"data": token["data"]}
yield {"type": "EmptyTag", "name": "meta",
"data": {(None, "charset"): self.encoding}}
yield {"type": "EndTag", "name": "head"}
meta_found = True
continue
elif type == "EndTag":
if token["name"].lower() == "head" and pending:
# insert meta into head (if necessary) and flush pending queue
yield pending.pop(0)
if not meta_found:
yield {"type": "EmptyTag", "name": "meta",
"data": {(None, "charset"): self.encoding}}
while pending:
yield pending.pop(0)
meta_found = True
state = "post_head"
if state == "in_head":
pending.append(token)
else:
yield token
@@ -0,0 +1,93 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
from . import base
from ..constants import namespaces, voidElements
from ..constants import spaceCharacters
spaceCharacters = "".join(spaceCharacters)
class Filter(base.Filter):
"""Lints the token stream for errors
If it finds any errors, it'll raise an ``AssertionError``.
"""
def __init__(self, source, require_matching_tags=True):
"""Creates a Filter
:arg source: the source token stream
:arg require_matching_tags: whether or not to require matching tags
"""
super(Filter, self).__init__(source)
self.require_matching_tags = require_matching_tags
def __iter__(self):
open_elements = []
for token in base.Filter.__iter__(self):
type = token["type"]
if type in ("StartTag", "EmptyTag"):
namespace = token["namespace"]
name = token["name"]
assert namespace is None or isinstance(namespace, text_type)
assert namespace != ""
assert isinstance(name, text_type)
assert name != ""
assert isinstance(token["data"], dict)
if (not namespace or namespace == namespaces["html"]) and name in voidElements:
assert type == "EmptyTag"
else:
assert type == "StartTag"
if type == "StartTag" and self.require_matching_tags:
open_elements.append((namespace, name))
for (namespace, name), value in token["data"].items():
assert namespace is None or isinstance(namespace, text_type)
assert namespace != ""
assert isinstance(name, text_type)
assert name != ""
assert isinstance(value, text_type)
elif type == "EndTag":
namespace = token["namespace"]
name = token["name"]
assert namespace is None or isinstance(namespace, text_type)
assert namespace != ""
assert isinstance(name, text_type)
assert name != ""
if (not namespace or namespace == namespaces["html"]) and name in voidElements:
assert False, "Void element reported as EndTag token: %(tag)s" % {"tag": name}
elif self.require_matching_tags:
start = open_elements.pop()
assert start == (namespace, name)
elif type == "Comment":
data = token["data"]
assert isinstance(data, text_type)
elif type in ("Characters", "SpaceCharacters"):
data = token["data"]
assert isinstance(data, text_type)
assert data != ""
if type == "SpaceCharacters":
assert data.strip(spaceCharacters) == ""
elif type == "Doctype":
name = token["name"]
assert name is None or isinstance(name, text_type)
assert token["publicId"] is None or isinstance(name, text_type)
assert token["systemId"] is None or isinstance(name, text_type)
elif type == "Entity":
assert isinstance(token["name"], text_type)
elif type == "SerializerError":
assert isinstance(token["data"], text_type)
else:
assert False, "Unknown token type: %(type)s" % {"type": type}
yield token
@@ -0,0 +1,207 @@
from __future__ import absolute_import, division, unicode_literals
from . import base
class Filter(base.Filter):
"""Removes optional tags from the token stream"""
def slider(self):
previous1 = previous2 = None
for token in self.source:
if previous1 is not None:
yield previous2, previous1, token
previous2 = previous1
previous1 = token
if previous1 is not None:
yield previous2, previous1, None
def __iter__(self):
for previous, token, next in self.slider():
type = token["type"]
if type == "StartTag":
if (token["data"] or
not self.is_optional_start(token["name"], previous, next)):
yield token
elif type == "EndTag":
if not self.is_optional_end(token["name"], next):
yield token
else:
yield token
def is_optional_start(self, tagname, previous, next):
type = next and next["type"] or None
if tagname in 'html':
# An html element's start tag may be omitted if the first thing
# inside the html element is not a space character or a comment.
return type not in ("Comment", "SpaceCharacters")
elif tagname == 'head':
# A head element's start tag may be omitted if the first thing
# inside the head element is an element.
# XXX: we also omit the start tag if the head element is empty
if type in ("StartTag", "EmptyTag"):
return True
elif type == "EndTag":
return next["name"] == "head"
elif tagname == 'body':
# A body element's start tag may be omitted if the first thing
# inside the body element is not a space character or a comment,
# except if the first thing inside the body element is a script
# or style element and the node immediately preceding the body
# element is a head element whose end tag has been omitted.
if type in ("Comment", "SpaceCharacters"):
return False
elif type == "StartTag":
# XXX: we do not look at the preceding event, so we never omit
# the body element's start tag if it's followed by a script or
# a style element.
return next["name"] not in ('script', 'style')
else:
return True
elif tagname == 'colgroup':
# A colgroup element's start tag may be omitted if the first thing
# inside the colgroup element is a col element, and if the element
# is not immediately preceded by another colgroup element whose
# end tag has been omitted.
if type in ("StartTag", "EmptyTag"):
# XXX: we do not look at the preceding event, so instead we never
# omit the colgroup element's end tag when it is immediately
# followed by another colgroup element. See is_optional_end.
return next["name"] == "col"
else:
return False
elif tagname == 'tbody':
# A tbody element's start tag may be omitted if the first thing
# inside the tbody element is a tr element, and if the element is
# not immediately preceded by a tbody, thead, or tfoot element
# whose end tag has been omitted.
if type == "StartTag":
# omit the thead and tfoot elements' end tag when they are
# immediately followed by a tbody element. See is_optional_end.
if previous and previous['type'] == 'EndTag' and \
previous['name'] in ('tbody', 'thead', 'tfoot'):
return False
return next["name"] == 'tr'
else:
return False
return False
def is_optional_end(self, tagname, next):
type = next and next["type"] or None
if tagname in ('html', 'head', 'body'):
# An html element's end tag may be omitted if the html element
# is not immediately followed by a space character or a comment.
return type not in ("Comment", "SpaceCharacters")
elif tagname in ('li', 'optgroup', 'tr'):
# A li element's end tag may be omitted if the li element is
# immediately followed by another li element or if there is
# no more content in the parent element.
# An optgroup element's end tag may be omitted if the optgroup
# element is immediately followed by another optgroup element,
# or if there is no more content in the parent element.
# A tr element's end tag may be omitted if the tr element is
# immediately followed by another tr element, or if there is
# no more content in the parent element.
if type == "StartTag":
return next["name"] == tagname
else:
return type == "EndTag" or type is None
elif tagname in ('dt', 'dd'):
# A dt element's end tag may be omitted if the dt element is
# immediately followed by another dt element or a dd element.
# A dd element's end tag may be omitted if the dd element is
# immediately followed by another dd element or a dt element,
# or if there is no more content in the parent element.
if type == "StartTag":
return next["name"] in ('dt', 'dd')
elif tagname == 'dd':
return type == "EndTag" or type is None
else:
return False
elif tagname == 'p':
# A p element's end tag may be omitted if the p element is
# immediately followed by an address, article, aside,
# blockquote, datagrid, dialog, dir, div, dl, fieldset,
# footer, form, h1, h2, h3, h4, h5, h6, header, hr, menu,
# nav, ol, p, pre, section, table, or ul, element, or if
# there is no more content in the parent element.
if type in ("StartTag", "EmptyTag"):
return next["name"] in ('address', 'article', 'aside',
'blockquote', 'datagrid', 'dialog',
'dir', 'div', 'dl', 'fieldset', 'footer',
'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
'header', 'hr', 'menu', 'nav', 'ol',
'p', 'pre', 'section', 'table', 'ul')
else:
return type == "EndTag" or type is None
elif tagname == 'option':
# An option element's end tag may be omitted if the option
# element is immediately followed by another option element,
# or if it is immediately followed by an <code>optgroup</code>
# element, or if there is no more content in the parent
# element.
if type == "StartTag":
return next["name"] in ('option', 'optgroup')
else:
return type == "EndTag" or type is None
elif tagname in ('rt', 'rp'):
# An rt element's end tag may be omitted if the rt element is
# immediately followed by an rt or rp element, or if there is
# no more content in the parent element.
# An rp element's end tag may be omitted if the rp element is
# immediately followed by an rt or rp element, or if there is
# no more content in the parent element.
if type == "StartTag":
return next["name"] in ('rt', 'rp')
else:
return type == "EndTag" or type is None
elif tagname == 'colgroup':
# A colgroup element's end tag may be omitted if the colgroup
# element is not immediately followed by a space character or
# a comment.
if type in ("Comment", "SpaceCharacters"):
return False
elif type == "StartTag":
# XXX: we also look for an immediately following colgroup
# element. See is_optional_start.
return next["name"] != 'colgroup'
else:
return True
elif tagname in ('thead', 'tbody'):
# A thead element's end tag may be omitted if the thead element
# is immediately followed by a tbody or tfoot element.
# A tbody element's end tag may be omitted if the tbody element
# is immediately followed by a tbody or tfoot element, or if
# there is no more content in the parent element.
# A tfoot element's end tag may be omitted if the tfoot element
# is immediately followed by a tbody element, or if there is no
# more content in the parent element.
# XXX: we never omit the end tag when the following element is
# a tbody. See is_optional_start.
if type == "StartTag":
return next["name"] in ['tbody', 'tfoot']
elif tagname == 'tbody':
return type == "EndTag" or type is None
else:
return False
elif tagname == 'tfoot':
# A tfoot element's end tag may be omitted if the tfoot element
# is immediately followed by a tbody element, or if there is no
# more content in the parent element.
# XXX: we never omit the end tag when the following element is
# a tbody. See is_optional_start.
if type == "StartTag":
return next["name"] == 'tbody'
else:
return type == "EndTag" or type is None
elif tagname in ('td', 'th'):
# A td element's end tag may be omitted if the td element is
# immediately followed by a td or th element, or if there is
# no more content in the parent element.
# A th element's end tag may be omitted if the th element is
# immediately followed by a td or th element, or if there is
# no more content in the parent element.
if type == "StartTag":
return next["name"] in ('td', 'th')
else:
return type == "EndTag" or type is None
return False
@@ -0,0 +1,916 @@
"""Deprecated from html5lib 1.1.
See `here <https://github.com/html5lib/html5lib-python/issues/443>`_ for
information about its deprecation; `Bleach <https://github.com/mozilla/bleach>`_
is recommended as a replacement. Please let us know in the aforementioned issue
if Bleach is unsuitable for your needs.
"""
from __future__ import absolute_import, division, unicode_literals
import re
import warnings
from xml.sax.saxutils import escape, unescape
from tensorboard._vendor.bleach.six_shim import urllib_parse as urlparse
from . import base
from ..constants import namespaces, prefixes
__all__ = ["Filter"]
_deprecation_msg = (
"html5lib's sanitizer is deprecated; see " +
"https://github.com/html5lib/html5lib-python/issues/443 and please let " +
"us know if Bleach is unsuitable for your needs"
)
warnings.warn(_deprecation_msg, DeprecationWarning)
allowed_elements = frozenset((
(namespaces['html'], 'a'),
(namespaces['html'], 'abbr'),
(namespaces['html'], 'acronym'),
(namespaces['html'], 'address'),
(namespaces['html'], 'area'),
(namespaces['html'], 'article'),
(namespaces['html'], 'aside'),
(namespaces['html'], 'audio'),
(namespaces['html'], 'b'),
(namespaces['html'], 'big'),
(namespaces['html'], 'blockquote'),
(namespaces['html'], 'br'),
(namespaces['html'], 'button'),
(namespaces['html'], 'canvas'),
(namespaces['html'], 'caption'),
(namespaces['html'], 'center'),
(namespaces['html'], 'cite'),
(namespaces['html'], 'code'),
(namespaces['html'], 'col'),
(namespaces['html'], 'colgroup'),
(namespaces['html'], 'command'),
(namespaces['html'], 'datagrid'),
(namespaces['html'], 'datalist'),
(namespaces['html'], 'dd'),
(namespaces['html'], 'del'),
(namespaces['html'], 'details'),
(namespaces['html'], 'dfn'),
(namespaces['html'], 'dialog'),
(namespaces['html'], 'dir'),
(namespaces['html'], 'div'),
(namespaces['html'], 'dl'),
(namespaces['html'], 'dt'),
(namespaces['html'], 'em'),
(namespaces['html'], 'event-source'),
(namespaces['html'], 'fieldset'),
(namespaces['html'], 'figcaption'),
(namespaces['html'], 'figure'),
(namespaces['html'], 'footer'),
(namespaces['html'], 'font'),
(namespaces['html'], 'form'),
(namespaces['html'], 'header'),
(namespaces['html'], 'h1'),
(namespaces['html'], 'h2'),
(namespaces['html'], 'h3'),
(namespaces['html'], 'h4'),
(namespaces['html'], 'h5'),
(namespaces['html'], 'h6'),
(namespaces['html'], 'hr'),
(namespaces['html'], 'i'),
(namespaces['html'], 'img'),
(namespaces['html'], 'input'),
(namespaces['html'], 'ins'),
(namespaces['html'], 'keygen'),
(namespaces['html'], 'kbd'),
(namespaces['html'], 'label'),
(namespaces['html'], 'legend'),
(namespaces['html'], 'li'),
(namespaces['html'], 'm'),
(namespaces['html'], 'map'),
(namespaces['html'], 'menu'),
(namespaces['html'], 'meter'),
(namespaces['html'], 'multicol'),
(namespaces['html'], 'nav'),
(namespaces['html'], 'nextid'),
(namespaces['html'], 'ol'),
(namespaces['html'], 'output'),
(namespaces['html'], 'optgroup'),
(namespaces['html'], 'option'),
(namespaces['html'], 'p'),
(namespaces['html'], 'pre'),
(namespaces['html'], 'progress'),
(namespaces['html'], 'q'),
(namespaces['html'], 's'),
(namespaces['html'], 'samp'),
(namespaces['html'], 'section'),
(namespaces['html'], 'select'),
(namespaces['html'], 'small'),
(namespaces['html'], 'sound'),
(namespaces['html'], 'source'),
(namespaces['html'], 'spacer'),
(namespaces['html'], 'span'),
(namespaces['html'], 'strike'),
(namespaces['html'], 'strong'),
(namespaces['html'], 'sub'),
(namespaces['html'], 'sup'),
(namespaces['html'], 'table'),
(namespaces['html'], 'tbody'),
(namespaces['html'], 'td'),
(namespaces['html'], 'textarea'),
(namespaces['html'], 'time'),
(namespaces['html'], 'tfoot'),
(namespaces['html'], 'th'),
(namespaces['html'], 'thead'),
(namespaces['html'], 'tr'),
(namespaces['html'], 'tt'),
(namespaces['html'], 'u'),
(namespaces['html'], 'ul'),
(namespaces['html'], 'var'),
(namespaces['html'], 'video'),
(namespaces['mathml'], 'maction'),
(namespaces['mathml'], 'math'),
(namespaces['mathml'], 'merror'),
(namespaces['mathml'], 'mfrac'),
(namespaces['mathml'], 'mi'),
(namespaces['mathml'], 'mmultiscripts'),
(namespaces['mathml'], 'mn'),
(namespaces['mathml'], 'mo'),
(namespaces['mathml'], 'mover'),
(namespaces['mathml'], 'mpadded'),
(namespaces['mathml'], 'mphantom'),
(namespaces['mathml'], 'mprescripts'),
(namespaces['mathml'], 'mroot'),
(namespaces['mathml'], 'mrow'),
(namespaces['mathml'], 'mspace'),
(namespaces['mathml'], 'msqrt'),
(namespaces['mathml'], 'mstyle'),
(namespaces['mathml'], 'msub'),
(namespaces['mathml'], 'msubsup'),
(namespaces['mathml'], 'msup'),
(namespaces['mathml'], 'mtable'),
(namespaces['mathml'], 'mtd'),
(namespaces['mathml'], 'mtext'),
(namespaces['mathml'], 'mtr'),
(namespaces['mathml'], 'munder'),
(namespaces['mathml'], 'munderover'),
(namespaces['mathml'], 'none'),
(namespaces['svg'], 'a'),
(namespaces['svg'], 'animate'),
(namespaces['svg'], 'animateColor'),
(namespaces['svg'], 'animateMotion'),
(namespaces['svg'], 'animateTransform'),
(namespaces['svg'], 'clipPath'),
(namespaces['svg'], 'circle'),
(namespaces['svg'], 'defs'),
(namespaces['svg'], 'desc'),
(namespaces['svg'], 'ellipse'),
(namespaces['svg'], 'font-face'),
(namespaces['svg'], 'font-face-name'),
(namespaces['svg'], 'font-face-src'),
(namespaces['svg'], 'g'),
(namespaces['svg'], 'glyph'),
(namespaces['svg'], 'hkern'),
(namespaces['svg'], 'linearGradient'),
(namespaces['svg'], 'line'),
(namespaces['svg'], 'marker'),
(namespaces['svg'], 'metadata'),
(namespaces['svg'], 'missing-glyph'),
(namespaces['svg'], 'mpath'),
(namespaces['svg'], 'path'),
(namespaces['svg'], 'polygon'),
(namespaces['svg'], 'polyline'),
(namespaces['svg'], 'radialGradient'),
(namespaces['svg'], 'rect'),
(namespaces['svg'], 'set'),
(namespaces['svg'], 'stop'),
(namespaces['svg'], 'svg'),
(namespaces['svg'], 'switch'),
(namespaces['svg'], 'text'),
(namespaces['svg'], 'title'),
(namespaces['svg'], 'tspan'),
(namespaces['svg'], 'use'),
))
allowed_attributes = frozenset((
# HTML attributes
(None, 'abbr'),
(None, 'accept'),
(None, 'accept-charset'),
(None, 'accesskey'),
(None, 'action'),
(None, 'align'),
(None, 'alt'),
(None, 'autocomplete'),
(None, 'autofocus'),
(None, 'axis'),
(None, 'background'),
(None, 'balance'),
(None, 'bgcolor'),
(None, 'bgproperties'),
(None, 'border'),
(None, 'bordercolor'),
(None, 'bordercolordark'),
(None, 'bordercolorlight'),
(None, 'bottompadding'),
(None, 'cellpadding'),
(None, 'cellspacing'),
(None, 'ch'),
(None, 'challenge'),
(None, 'char'),
(None, 'charoff'),
(None, 'choff'),
(None, 'charset'),
(None, 'checked'),
(None, 'cite'),
(None, 'class'),
(None, 'clear'),
(None, 'color'),
(None, 'cols'),
(None, 'colspan'),
(None, 'compact'),
(None, 'contenteditable'),
(None, 'controls'),
(None, 'coords'),
(None, 'data'),
(None, 'datafld'),
(None, 'datapagesize'),
(None, 'datasrc'),
(None, 'datetime'),
(None, 'default'),
(None, 'delay'),
(None, 'dir'),
(None, 'disabled'),
(None, 'draggable'),
(None, 'dynsrc'),
(None, 'enctype'),
(None, 'end'),
(None, 'face'),
(None, 'for'),
(None, 'form'),
(None, 'frame'),
(None, 'galleryimg'),
(None, 'gutter'),
(None, 'headers'),
(None, 'height'),
(None, 'hidefocus'),
(None, 'hidden'),
(None, 'high'),
(None, 'href'),
(None, 'hreflang'),
(None, 'hspace'),
(None, 'icon'),
(None, 'id'),
(None, 'inputmode'),
(None, 'ismap'),
(None, 'keytype'),
(None, 'label'),
(None, 'leftspacing'),
(None, 'lang'),
(None, 'list'),
(None, 'longdesc'),
(None, 'loop'),
(None, 'loopcount'),
(None, 'loopend'),
(None, 'loopstart'),
(None, 'low'),
(None, 'lowsrc'),
(None, 'max'),
(None, 'maxlength'),
(None, 'media'),
(None, 'method'),
(None, 'min'),
(None, 'multiple'),
(None, 'name'),
(None, 'nohref'),
(None, 'noshade'),
(None, 'nowrap'),
(None, 'open'),
(None, 'optimum'),
(None, 'pattern'),
(None, 'ping'),
(None, 'point-size'),
(None, 'poster'),
(None, 'pqg'),
(None, 'preload'),
(None, 'prompt'),
(None, 'radiogroup'),
(None, 'readonly'),
(None, 'rel'),
(None, 'repeat-max'),
(None, 'repeat-min'),
(None, 'replace'),
(None, 'required'),
(None, 'rev'),
(None, 'rightspacing'),
(None, 'rows'),
(None, 'rowspan'),
(None, 'rules'),
(None, 'scope'),
(None, 'selected'),
(None, 'shape'),
(None, 'size'),
(None, 'span'),
(None, 'src'),
(None, 'start'),
(None, 'step'),
(None, 'style'),
(None, 'summary'),
(None, 'suppress'),
(None, 'tabindex'),
(None, 'target'),
(None, 'template'),
(None, 'title'),
(None, 'toppadding'),
(None, 'type'),
(None, 'unselectable'),
(None, 'usemap'),
(None, 'urn'),
(None, 'valign'),
(None, 'value'),
(None, 'variable'),
(None, 'volume'),
(None, 'vspace'),
(None, 'vrml'),
(None, 'width'),
(None, 'wrap'),
(namespaces['xml'], 'lang'),
# MathML attributes
(None, 'actiontype'),
(None, 'align'),
(None, 'columnalign'),
(None, 'columnalign'),
(None, 'columnalign'),
(None, 'columnlines'),
(None, 'columnspacing'),
(None, 'columnspan'),
(None, 'depth'),
(None, 'display'),
(None, 'displaystyle'),
(None, 'equalcolumns'),
(None, 'equalrows'),
(None, 'fence'),
(None, 'fontstyle'),
(None, 'fontweight'),
(None, 'frame'),
(None, 'height'),
(None, 'linethickness'),
(None, 'lspace'),
(None, 'mathbackground'),
(None, 'mathcolor'),
(None, 'mathvariant'),
(None, 'mathvariant'),
(None, 'maxsize'),
(None, 'minsize'),
(None, 'other'),
(None, 'rowalign'),
(None, 'rowalign'),
(None, 'rowalign'),
(None, 'rowlines'),
(None, 'rowspacing'),
(None, 'rowspan'),
(None, 'rspace'),
(None, 'scriptlevel'),
(None, 'selection'),
(None, 'separator'),
(None, 'stretchy'),
(None, 'width'),
(None, 'width'),
(namespaces['xlink'], 'href'),
(namespaces['xlink'], 'show'),
(namespaces['xlink'], 'type'),
# SVG attributes
(None, 'accent-height'),
(None, 'accumulate'),
(None, 'additive'),
(None, 'alphabetic'),
(None, 'arabic-form'),
(None, 'ascent'),
(None, 'attributeName'),
(None, 'attributeType'),
(None, 'baseProfile'),
(None, 'bbox'),
(None, 'begin'),
(None, 'by'),
(None, 'calcMode'),
(None, 'cap-height'),
(None, 'class'),
(None, 'clip-path'),
(None, 'color'),
(None, 'color-rendering'),
(None, 'content'),
(None, 'cx'),
(None, 'cy'),
(None, 'd'),
(None, 'dx'),
(None, 'dy'),
(None, 'descent'),
(None, 'display'),
(None, 'dur'),
(None, 'end'),
(None, 'fill'),
(None, 'fill-opacity'),
(None, 'fill-rule'),
(None, 'font-family'),
(None, 'font-size'),
(None, 'font-stretch'),
(None, 'font-style'),
(None, 'font-variant'),
(None, 'font-weight'),
(None, 'from'),
(None, 'fx'),
(None, 'fy'),
(None, 'g1'),
(None, 'g2'),
(None, 'glyph-name'),
(None, 'gradientUnits'),
(None, 'hanging'),
(None, 'height'),
(None, 'horiz-adv-x'),
(None, 'horiz-origin-x'),
(None, 'id'),
(None, 'ideographic'),
(None, 'k'),
(None, 'keyPoints'),
(None, 'keySplines'),
(None, 'keyTimes'),
(None, 'lang'),
(None, 'marker-end'),
(None, 'marker-mid'),
(None, 'marker-start'),
(None, 'markerHeight'),
(None, 'markerUnits'),
(None, 'markerWidth'),
(None, 'mathematical'),
(None, 'max'),
(None, 'min'),
(None, 'name'),
(None, 'offset'),
(None, 'opacity'),
(None, 'orient'),
(None, 'origin'),
(None, 'overline-position'),
(None, 'overline-thickness'),
(None, 'panose-1'),
(None, 'path'),
(None, 'pathLength'),
(None, 'points'),
(None, 'preserveAspectRatio'),
(None, 'r'),
(None, 'refX'),
(None, 'refY'),
(None, 'repeatCount'),
(None, 'repeatDur'),
(None, 'requiredExtensions'),
(None, 'requiredFeatures'),
(None, 'restart'),
(None, 'rotate'),
(None, 'rx'),
(None, 'ry'),
(None, 'slope'),
(None, 'stemh'),
(None, 'stemv'),
(None, 'stop-color'),
(None, 'stop-opacity'),
(None, 'strikethrough-position'),
(None, 'strikethrough-thickness'),
(None, 'stroke'),
(None, 'stroke-dasharray'),
(None, 'stroke-dashoffset'),
(None, 'stroke-linecap'),
(None, 'stroke-linejoin'),
(None, 'stroke-miterlimit'),
(None, 'stroke-opacity'),
(None, 'stroke-width'),
(None, 'systemLanguage'),
(None, 'target'),
(None, 'text-anchor'),
(None, 'to'),
(None, 'transform'),
(None, 'type'),
(None, 'u1'),
(None, 'u2'),
(None, 'underline-position'),
(None, 'underline-thickness'),
(None, 'unicode'),
(None, 'unicode-range'),
(None, 'units-per-em'),
(None, 'values'),
(None, 'version'),
(None, 'viewBox'),
(None, 'visibility'),
(None, 'width'),
(None, 'widths'),
(None, 'x'),
(None, 'x-height'),
(None, 'x1'),
(None, 'x2'),
(namespaces['xlink'], 'actuate'),
(namespaces['xlink'], 'arcrole'),
(namespaces['xlink'], 'href'),
(namespaces['xlink'], 'role'),
(namespaces['xlink'], 'show'),
(namespaces['xlink'], 'title'),
(namespaces['xlink'], 'type'),
(namespaces['xml'], 'base'),
(namespaces['xml'], 'lang'),
(namespaces['xml'], 'space'),
(None, 'y'),
(None, 'y1'),
(None, 'y2'),
(None, 'zoomAndPan'),
))
attr_val_is_uri = frozenset((
(None, 'href'),
(None, 'src'),
(None, 'cite'),
(None, 'action'),
(None, 'longdesc'),
(None, 'poster'),
(None, 'background'),
(None, 'datasrc'),
(None, 'dynsrc'),
(None, 'lowsrc'),
(None, 'ping'),
(namespaces['xlink'], 'href'),
(namespaces['xml'], 'base'),
))
svg_attr_val_allows_ref = frozenset((
(None, 'clip-path'),
(None, 'color-profile'),
(None, 'cursor'),
(None, 'fill'),
(None, 'filter'),
(None, 'marker'),
(None, 'marker-start'),
(None, 'marker-mid'),
(None, 'marker-end'),
(None, 'mask'),
(None, 'stroke'),
))
svg_allow_local_href = frozenset((
(None, 'altGlyph'),
(None, 'animate'),
(None, 'animateColor'),
(None, 'animateMotion'),
(None, 'animateTransform'),
(None, 'cursor'),
(None, 'feImage'),
(None, 'filter'),
(None, 'linearGradient'),
(None, 'pattern'),
(None, 'radialGradient'),
(None, 'textpath'),
(None, 'tref'),
(None, 'set'),
(None, 'use')
))
allowed_css_properties = frozenset((
'azimuth',
'background-color',
'border-bottom-color',
'border-collapse',
'border-color',
'border-left-color',
'border-right-color',
'border-top-color',
'clear',
'color',
'cursor',
'direction',
'display',
'elevation',
'float',
'font',
'font-family',
'font-size',
'font-style',
'font-variant',
'font-weight',
'height',
'letter-spacing',
'line-height',
'overflow',
'pause',
'pause-after',
'pause-before',
'pitch',
'pitch-range',
'richness',
'speak',
'speak-header',
'speak-numeral',
'speak-punctuation',
'speech-rate',
'stress',
'text-align',
'text-decoration',
'text-indent',
'unicode-bidi',
'vertical-align',
'voice-family',
'volume',
'white-space',
'width',
))
allowed_css_keywords = frozenset((
'auto',
'aqua',
'black',
'block',
'blue',
'bold',
'both',
'bottom',
'brown',
'center',
'collapse',
'dashed',
'dotted',
'fuchsia',
'gray',
'green',
'!important',
'italic',
'left',
'lime',
'maroon',
'medium',
'none',
'navy',
'normal',
'nowrap',
'olive',
'pointer',
'purple',
'red',
'right',
'solid',
'silver',
'teal',
'top',
'transparent',
'underline',
'white',
'yellow',
))
allowed_svg_properties = frozenset((
'fill',
'fill-opacity',
'fill-rule',
'stroke',
'stroke-width',
'stroke-linecap',
'stroke-linejoin',
'stroke-opacity',
))
allowed_protocols = frozenset((
'ed2k',
'ftp',
'http',
'https',
'irc',
'mailto',
'news',
'gopher',
'nntp',
'telnet',
'webcal',
'xmpp',
'callto',
'feed',
'urn',
'aim',
'rsync',
'tag',
'ssh',
'sftp',
'rtsp',
'afs',
'data',
))
allowed_content_types = frozenset((
'image/png',
'image/jpeg',
'image/gif',
'image/webp',
'image/bmp',
'text/plain',
))
data_content_type = re.compile(r'''
^
# Match a content type <application>/<type>
(?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
# Match any character set and encoding
(?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
|(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
# Assume the rest is data
,.*
$
''',
re.VERBOSE)
class Filter(base.Filter):
"""Sanitizes token stream of XHTML+MathML+SVG and of inline style attributes"""
def __init__(self,
source,
allowed_elements=allowed_elements,
allowed_attributes=allowed_attributes,
allowed_css_properties=allowed_css_properties,
allowed_css_keywords=allowed_css_keywords,
allowed_svg_properties=allowed_svg_properties,
allowed_protocols=allowed_protocols,
allowed_content_types=allowed_content_types,
attr_val_is_uri=attr_val_is_uri,
svg_attr_val_allows_ref=svg_attr_val_allows_ref,
svg_allow_local_href=svg_allow_local_href):
"""Creates a Filter
:arg allowed_elements: set of elements to allow--everything else will
be escaped
:arg allowed_attributes: set of attributes to allow in
elements--everything else will be stripped
:arg allowed_css_properties: set of CSS properties to allow--everything
else will be stripped
:arg allowed_css_keywords: set of CSS keywords to allow--everything
else will be stripped
:arg allowed_svg_properties: set of SVG properties to allow--everything
else will be removed
:arg allowed_protocols: set of allowed protocols for URIs
:arg allowed_content_types: set of allowed content types for ``data`` URIs.
:arg attr_val_is_uri: set of attributes that have URI values--values
that have a scheme not listed in ``allowed_protocols`` are removed
:arg svg_attr_val_allows_ref: set of SVG attributes that can have
references
:arg svg_allow_local_href: set of SVG elements that can have local
hrefs--these are removed
"""
super(Filter, self).__init__(source)
warnings.warn(_deprecation_msg, DeprecationWarning)
self.allowed_elements = allowed_elements
self.allowed_attributes = allowed_attributes
self.allowed_css_properties = allowed_css_properties
self.allowed_css_keywords = allowed_css_keywords
self.allowed_svg_properties = allowed_svg_properties
self.allowed_protocols = allowed_protocols
self.allowed_content_types = allowed_content_types
self.attr_val_is_uri = attr_val_is_uri
self.svg_attr_val_allows_ref = svg_attr_val_allows_ref
self.svg_allow_local_href = svg_allow_local_href
def __iter__(self):
for token in base.Filter.__iter__(self):
token = self.sanitize_token(token)
if token:
yield token
# Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
# stripping out all attributes not in ALLOWED_ATTRIBUTES. Style attributes
# are parsed, and a restricted set, specified by ALLOWED_CSS_PROPERTIES and
# ALLOWED_CSS_KEYWORDS, are allowed through. attributes in ATTR_VAL_IS_URI
# are scanned, and only URI schemes specified in ALLOWED_PROTOCOLS are
# allowed.
#
# sanitize_html('<script> do_nasty_stuff() </script>')
# => &lt;script> do_nasty_stuff() &lt;/script>
# sanitize_html('<a href="javascript: sucker();">Click here for $100</a>')
# => <a>Click here for $100</a>
def sanitize_token(self, token):
# accommodate filters which use token_type differently
token_type = token["type"]
if token_type in ("StartTag", "EndTag", "EmptyTag"):
name = token["name"]
namespace = token["namespace"]
if ((namespace, name) in self.allowed_elements or
(namespace is None and
(namespaces["html"], name) in self.allowed_elements)):
return self.allowed_token(token)
else:
return self.disallowed_token(token)
elif token_type == "Comment":
pass
else:
return token
def allowed_token(self, token):
if "data" in token:
attrs = token["data"]
attr_names = set(attrs.keys())
# Remove forbidden attributes
for to_remove in (attr_names - self.allowed_attributes):
del token["data"][to_remove]
attr_names.remove(to_remove)
# Remove attributes with disallowed URL values
for attr in (attr_names & self.attr_val_is_uri):
assert attr in attrs
# I don't have a clue where this regexp comes from or why it matches those
# characters, nor why we call unescape. I just know it's always been here.
# Should you be worried by this comment in a sanitizer? Yes. On the other hand, all
# this will do is remove *more* than it otherwise would.
val_unescaped = re.sub("[`\x00-\x20\x7f-\xa0\\s]+", '',
unescape(attrs[attr])).lower()
# remove replacement characters from unescaped characters
val_unescaped = val_unescaped.replace("\ufffd", "")
try:
uri = urlparse.urlparse(val_unescaped)
except ValueError:
uri = None
del attrs[attr]
if uri and uri.scheme:
if uri.scheme not in self.allowed_protocols:
del attrs[attr]
if uri.scheme == 'data':
m = data_content_type.match(uri.path)
if not m:
del attrs[attr]
elif m.group('content_type') not in self.allowed_content_types:
del attrs[attr]
for attr in self.svg_attr_val_allows_ref:
if attr in attrs:
attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
' ',
unescape(attrs[attr]))
if (token["name"] in self.svg_allow_local_href and
(namespaces['xlink'], 'href') in attrs and re.search(r'^\s*[^#\s].*',
attrs[(namespaces['xlink'], 'href')])):
del attrs[(namespaces['xlink'], 'href')]
if (None, 'style') in attrs:
attrs[(None, 'style')] = self.sanitize_css(attrs[(None, 'style')])
token["data"] = attrs
return token
def disallowed_token(self, token):
token_type = token["type"]
if token_type == "EndTag":
token["data"] = "</%s>" % token["name"]
elif token["data"]:
assert token_type in ("StartTag", "EmptyTag")
attrs = []
for (ns, name), v in token["data"].items():
attrs.append(' %s="%s"' % (name if ns is None else "%s:%s" % (prefixes[ns], name), escape(v)))
token["data"] = "<%s%s>" % (token["name"], ''.join(attrs))
else:
token["data"] = "<%s>" % token["name"]
if token.get("selfClosing"):
token["data"] = token["data"][:-1] + "/>"
token["type"] = "Characters"
del token["name"]
return token
def sanitize_css(self, style):
# disallow urls
style = re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
# gauntlet
if not re.match(r"""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
return ''
if not re.match(r"^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style):
return ''
clean = []
for prop, value in re.findall(r"([-\w]+)\s*:\s*([^:;]*)", style):
if not value:
continue
if prop.lower() in self.allowed_css_properties:
clean.append(prop + ': ' + value + ';')
elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
'padding']:
for keyword in value.split():
if keyword not in self.allowed_css_keywords and \
not re.match(r"^(#[0-9a-fA-F]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): # noqa
break
else:
clean.append(prop + ': ' + value + ';')
elif prop.lower() in self.allowed_svg_properties:
clean.append(prop + ': ' + value + ';')
return ' '.join(clean)
@@ -0,0 +1,38 @@
from __future__ import absolute_import, division, unicode_literals
import re
from . import base
from ..constants import rcdataElements, spaceCharacters
spaceCharacters = "".join(spaceCharacters)
SPACES_REGEX = re.compile("[%s]+" % spaceCharacters)
class Filter(base.Filter):
"""Collapses whitespace except in pre, textarea, and script elements"""
spacePreserveElements = frozenset(["pre", "textarea"] + list(rcdataElements))
def __iter__(self):
preserve = 0
for token in base.Filter.__iter__(self):
type = token["type"]
if type == "StartTag" \
and (preserve or token["name"] in self.spacePreserveElements):
preserve += 1
elif type == "EndTag" and preserve:
preserve -= 1
elif not preserve and type == "SpaceCharacters" and token["data"]:
# Test on token["data"] above to not introduce spaces where there were not
token["data"] = " "
elif not preserve and type == "Characters":
token["data"] = collapse_spaces(token["data"])
yield token
def collapse_spaces(text):
return SPACES_REGEX.sub(' ', text)
@@ -0,0 +1,409 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
import re
from codecs import register_error, xmlcharrefreplace_errors
from .constants import voidElements, booleanAttributes, spaceCharacters
from .constants import rcdataElements, entities, xmlEntities
from . import treewalkers, _utils
from xml.sax.saxutils import escape
_quoteAttributeSpecChars = "".join(spaceCharacters) + "\"'=<>`"
_quoteAttributeSpec = re.compile("[" + _quoteAttributeSpecChars + "]")
_quoteAttributeLegacy = re.compile("[" + _quoteAttributeSpecChars +
"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n"
"\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15"
"\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
"\x20\x2f\x60\xa0\u1680\u180e\u180f\u2000"
"\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
"\u2008\u2009\u200a\u2028\u2029\u202f\u205f"
"\u3000]")
_encode_entity_map = {}
_is_ucs4 = len("\U0010FFFF") == 1
for k, v in list(entities.items()):
# skip multi-character entities
if ((_is_ucs4 and len(v) > 1) or
(not _is_ucs4 and len(v) > 2)):
continue
if v != "&":
if len(v) == 2:
v = _utils.surrogatePairToCodepoint(v)
else:
v = ord(v)
if v not in _encode_entity_map or k.islower():
# prefer &lt; over &LT; and similarly for &amp;, &gt;, etc.
_encode_entity_map[v] = k
def htmlentityreplace_errors(exc):
if isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)):
res = []
codepoints = []
skip = False
for i, c in enumerate(exc.object[exc.start:exc.end]):
if skip:
skip = False
continue
index = i + exc.start
if _utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]):
codepoint = _utils.surrogatePairToCodepoint(exc.object[index:index + 2])
skip = True
else:
codepoint = ord(c)
codepoints.append(codepoint)
for cp in codepoints:
e = _encode_entity_map.get(cp)
if e:
res.append("&")
res.append(e)
if not e.endswith(";"):
res.append(";")
else:
res.append("&#x%s;" % (hex(cp)[2:]))
return ("".join(res), exc.end)
else:
return xmlcharrefreplace_errors(exc)
register_error("htmlentityreplace", htmlentityreplace_errors)
def serialize(input, tree="etree", encoding=None, **serializer_opts):
"""Serializes the input token stream using the specified treewalker
:arg input: the token stream to serialize
:arg tree: the treewalker to use
:arg encoding: the encoding to use
:arg serializer_opts: any options to pass to the
:py:class:`html5lib.serializer.HTMLSerializer` that gets created
:returns: the tree serialized as a string
Example:
>>> from html5lib.html5parser import parse
>>> from html5lib.serializer import serialize
>>> token_stream = parse('<html><body><p>Hi!</p></body></html>')
>>> serialize(token_stream, omit_optional_tags=False)
'<html><head></head><body><p>Hi!</p></body></html>'
"""
# XXX: Should we cache this?
walker = treewalkers.getTreeWalker(tree)
s = HTMLSerializer(**serializer_opts)
return s.render(walker(input), encoding)
class HTMLSerializer(object):
# attribute quoting options
quote_attr_values = "legacy" # be secure by default
quote_char = '"'
use_best_quote_char = True
# tag syntax options
omit_optional_tags = True
minimize_boolean_attributes = True
use_trailing_solidus = False
space_before_trailing_solidus = True
# escaping options
escape_lt_in_attrs = False
escape_rcdata = False
resolve_entities = True
# miscellaneous options
alphabetical_attributes = False
inject_meta_charset = True
strip_whitespace = False
sanitize = False
options = ("quote_attr_values", "quote_char", "use_best_quote_char",
"omit_optional_tags", "minimize_boolean_attributes",
"use_trailing_solidus", "space_before_trailing_solidus",
"escape_lt_in_attrs", "escape_rcdata", "resolve_entities",
"alphabetical_attributes", "inject_meta_charset",
"strip_whitespace", "sanitize")
def __init__(self, **kwargs):
"""Initialize HTMLSerializer
:arg inject_meta_charset: Whether or not to inject the meta charset.
Defaults to ``True``.
:arg quote_attr_values: Whether to quote attribute values that don't
require quoting per legacy browser behavior (``"legacy"``), when
required by the standard (``"spec"``), or always (``"always"``).
Defaults to ``"legacy"``.
:arg quote_char: Use given quote character for attribute quoting.
Defaults to ``"`` which will use double quotes unless attribute
value contains a double quote, in which case single quotes are
used.
:arg escape_lt_in_attrs: Whether or not to escape ``<`` in attribute
values.
Defaults to ``False``.
:arg escape_rcdata: Whether to escape characters that need to be
escaped within normal elements within rcdata elements such as
style.
Defaults to ``False``.
:arg resolve_entities: Whether to resolve named character entities that
appear in the source tree. The XML predefined entities &lt; &gt;
&amp; &quot; &apos; are unaffected by this setting.
Defaults to ``True``.
:arg strip_whitespace: Whether to remove semantically meaningless
whitespace. (This compresses all whitespace to a single space
except within ``pre``.)
Defaults to ``False``.
:arg minimize_boolean_attributes: Shortens boolean attributes to give
just the attribute value, for example::
<input disabled="disabled">
becomes::
<input disabled>
Defaults to ``True``.
:arg use_trailing_solidus: Includes a close-tag slash at the end of the
start tag of void elements (empty elements whose end tag is
forbidden). E.g. ``<hr/>``.
Defaults to ``False``.
:arg space_before_trailing_solidus: Places a space immediately before
the closing slash in a tag using a trailing solidus. E.g.
``<hr />``. Requires ``use_trailing_solidus=True``.
Defaults to ``True``.
:arg sanitize: Strip all unsafe or unknown constructs from output.
See :py:class:`html5lib.filters.sanitizer.Filter`.
Defaults to ``False``.
:arg omit_optional_tags: Omit start/end tags that are optional.
Defaults to ``True``.
:arg alphabetical_attributes: Reorder attributes to be in alphabetical order.
Defaults to ``False``.
"""
unexpected_args = frozenset(kwargs) - frozenset(self.options)
if len(unexpected_args) > 0:
raise TypeError("__init__() got an unexpected keyword argument '%s'" % next(iter(unexpected_args)))
if 'quote_char' in kwargs:
self.use_best_quote_char = False
for attr in self.options:
setattr(self, attr, kwargs.get(attr, getattr(self, attr)))
self.errors = []
self.strict = False
def encode(self, string):
assert(isinstance(string, text_type))
if self.encoding:
return string.encode(self.encoding, "htmlentityreplace")
else:
return string
def encodeStrict(self, string):
assert(isinstance(string, text_type))
if self.encoding:
return string.encode(self.encoding, "strict")
else:
return string
def serialize(self, treewalker, encoding=None):
# pylint:disable=too-many-nested-blocks
self.encoding = encoding
in_cdata = False
self.errors = []
if encoding and self.inject_meta_charset:
from .filters.inject_meta_charset import Filter
treewalker = Filter(treewalker, encoding)
# Alphabetical attributes is here under the assumption that none of
# the later filters add or change order of attributes; it needs to be
# before the sanitizer so escaped elements come out correctly
if self.alphabetical_attributes:
from .filters.alphabeticalattributes import Filter
treewalker = Filter(treewalker)
# WhitespaceFilter should be used before OptionalTagFilter
# for maximum efficiently of this latter filter
if self.strip_whitespace:
from .filters.whitespace import Filter
treewalker = Filter(treewalker)
if self.sanitize:
from .filters.sanitizer import Filter
treewalker = Filter(treewalker)
if self.omit_optional_tags:
from .filters.optionaltags import Filter
treewalker = Filter(treewalker)
for token in treewalker:
type = token["type"]
if type == "Doctype":
doctype = "<!DOCTYPE %s" % token["name"]
if token["publicId"]:
doctype += ' PUBLIC "%s"' % token["publicId"]
elif token["systemId"]:
doctype += " SYSTEM"
if token["systemId"]:
if token["systemId"].find('"') >= 0:
if token["systemId"].find("'") >= 0:
self.serializeError("System identifier contains both single and double quote characters")
quote_char = "'"
else:
quote_char = '"'
doctype += " %s%s%s" % (quote_char, token["systemId"], quote_char)
doctype += ">"
yield self.encodeStrict(doctype)
elif type in ("Characters", "SpaceCharacters"):
if type == "SpaceCharacters" or in_cdata:
if in_cdata and token["data"].find("</") >= 0:
self.serializeError("Unexpected </ in CDATA")
yield self.encode(token["data"])
else:
yield self.encode(escape(token["data"]))
elif type in ("StartTag", "EmptyTag"):
name = token["name"]
yield self.encodeStrict("<%s" % name)
if name in rcdataElements and not self.escape_rcdata:
in_cdata = True
elif in_cdata:
self.serializeError("Unexpected child element of a CDATA element")
for (_, attr_name), attr_value in token["data"].items():
# TODO: Add namespace support here
k = attr_name
v = attr_value
yield self.encodeStrict(' ')
yield self.encodeStrict(k)
if not self.minimize_boolean_attributes or \
(k not in booleanAttributes.get(name, tuple()) and
k not in booleanAttributes.get("", tuple())):
yield self.encodeStrict("=")
if self.quote_attr_values == "always" or len(v) == 0:
quote_attr = True
elif self.quote_attr_values == "spec":
quote_attr = _quoteAttributeSpec.search(v) is not None
elif self.quote_attr_values == "legacy":
quote_attr = _quoteAttributeLegacy.search(v) is not None
else:
raise ValueError("quote_attr_values must be one of: "
"'always', 'spec', or 'legacy'")
v = v.replace("&", "&amp;")
if self.escape_lt_in_attrs:
v = v.replace("<", "&lt;")
if quote_attr:
quote_char = self.quote_char
if self.use_best_quote_char:
if "'" in v and '"' not in v:
quote_char = '"'
elif '"' in v and "'" not in v:
quote_char = "'"
if quote_char == "'":
v = v.replace("'", "&#39;")
else:
v = v.replace('"', "&quot;")
yield self.encodeStrict(quote_char)
yield self.encode(v)
yield self.encodeStrict(quote_char)
else:
yield self.encode(v)
if name in voidElements and self.use_trailing_solidus:
if self.space_before_trailing_solidus:
yield self.encodeStrict(" /")
else:
yield self.encodeStrict("/")
yield self.encode(">")
elif type == "EndTag":
name = token["name"]
if name in rcdataElements:
in_cdata = False
elif in_cdata:
self.serializeError("Unexpected child element of a CDATA element")
yield self.encodeStrict("</%s>" % name)
elif type == "Comment":
data = token["data"]
if data.find("--") >= 0:
self.serializeError("Comment contains --")
yield self.encodeStrict("<!--%s-->" % token["data"])
elif type == "Entity":
name = token["name"]
key = name + ";"
if key not in entities:
self.serializeError("Entity %s not recognized" % name)
if self.resolve_entities and key not in xmlEntities:
data = entities[key]
else:
data = "&%s;" % name
yield self.encodeStrict(data)
else:
self.serializeError(token["data"])
def render(self, treewalker, encoding=None):
"""Serializes the stream from the treewalker into a string
:arg treewalker: the treewalker to serialize
:arg encoding: the string encoding to use
:returns: the serialized tree
Example:
>>> from html5lib import parse, getTreeWalker
>>> from html5lib.serializer import HTMLSerializer
>>> token_stream = parse('<html><body>Hi!</body></html>')
>>> walker = getTreeWalker('etree')
>>> serializer = HTMLSerializer(omit_optional_tags=False)
>>> serializer.render(walker(token_stream))
'<html><head></head><body>Hi!</body></html>'
"""
if encoding:
return b"".join(list(self.serialize(treewalker, encoding)))
else:
return "".join(list(self.serialize(treewalker)))
def serializeError(self, data="XXX ERROR MESSAGE NEEDED"):
# XXX The idea is to make data mandatory.
self.errors.append(data)
if self.strict:
raise SerializeError
class SerializeError(Exception):
"""Error in serialized tree"""
pass
@@ -0,0 +1,30 @@
"""Tree adapters let you convert from one tree structure to another
Example:
.. code-block:: python
import html5lib
from html5lib.treeadapters import genshi
doc = '<html><body>Hi!</body></html>'
treebuilder = html5lib.getTreeBuilder('etree')
parser = html5lib.HTMLParser(tree=treebuilder)
tree = parser.parse(doc)
TreeWalker = html5lib.getTreeWalker('etree')
genshi_tree = genshi.to_genshi(TreeWalker(tree))
"""
from __future__ import absolute_import, division, unicode_literals
from . import sax
__all__ = ["sax"]
try:
from . import genshi # noqa
except ImportError:
pass
else:
__all__.append("genshi")
@@ -0,0 +1,54 @@
from __future__ import absolute_import, division, unicode_literals
from genshi.core import QName, Attrs
from genshi.core import START, END, TEXT, COMMENT, DOCTYPE
def to_genshi(walker):
"""Convert a tree to a genshi tree
:arg walker: the treewalker to use to walk the tree to convert it
:returns: generator of genshi nodes
"""
text = []
for token in walker:
type = token["type"]
if type in ("Characters", "SpaceCharacters"):
text.append(token["data"])
elif text:
yield TEXT, "".join(text), (None, -1, -1)
text = []
if type in ("StartTag", "EmptyTag"):
if token["namespace"]:
name = "{%s}%s" % (token["namespace"], token["name"])
else:
name = token["name"]
attrs = Attrs([(QName("{%s}%s" % attr if attr[0] is not None else attr[1]), value)
for attr, value in token["data"].items()])
yield (START, (QName(name), attrs), (None, -1, -1))
if type == "EmptyTag":
type = "EndTag"
if type == "EndTag":
if token["namespace"]:
name = "{%s}%s" % (token["namespace"], token["name"])
else:
name = token["name"]
yield END, QName(name), (None, -1, -1)
elif type == "Comment":
yield COMMENT, token["data"], (None, -1, -1)
elif type == "Doctype":
yield DOCTYPE, (token["name"], token["publicId"],
token["systemId"]), (None, -1, -1)
else:
pass # FIXME: What to do?
if text:
yield TEXT, "".join(text), (None, -1, -1)
@@ -0,0 +1,50 @@
from __future__ import absolute_import, division, unicode_literals
from xml.sax.xmlreader import AttributesNSImpl
from ..constants import adjustForeignAttributes, unadjustForeignAttributes
prefix_mapping = {}
for prefix, localName, namespace in adjustForeignAttributes.values():
if prefix is not None:
prefix_mapping[prefix] = namespace
def to_sax(walker, handler):
"""Call SAX-like content handler based on treewalker walker
:arg walker: the treewalker to use to walk the tree to convert it
:arg handler: SAX handler to use
"""
handler.startDocument()
for prefix, namespace in prefix_mapping.items():
handler.startPrefixMapping(prefix, namespace)
for token in walker:
type = token["type"]
if type == "Doctype":
continue
elif type in ("StartTag", "EmptyTag"):
attrs = AttributesNSImpl(token["data"],
unadjustForeignAttributes)
handler.startElementNS((token["namespace"], token["name"]),
token["name"],
attrs)
if type == "EmptyTag":
handler.endElementNS((token["namespace"], token["name"]),
token["name"])
elif type == "EndTag":
handler.endElementNS((token["namespace"], token["name"]),
token["name"])
elif type in ("Characters", "SpaceCharacters"):
handler.characters(token["data"])
elif type == "Comment":
pass
else:
assert False, "Unknown token type"
for prefix, namespace in prefix_mapping.items():
handler.endPrefixMapping(prefix)
handler.endDocument()
@@ -0,0 +1,88 @@
"""A collection of modules for building different kinds of trees from HTML
documents.
To create a treebuilder for a new type of tree, you need to do
implement several things:
1. A set of classes for various types of elements: Document, Doctype, Comment,
Element. These must implement the interface of ``base.treebuilders.Node``
(although comment nodes have a different signature for their constructor,
see ``treebuilders.etree.Comment``) Textual content may also be implemented
as another node type, or not, as your tree implementation requires.
2. A treebuilder object (called ``TreeBuilder`` by convention) that inherits
from ``treebuilders.base.TreeBuilder``. This has 4 required attributes:
* ``documentClass`` - the class to use for the bottommost node of a document
* ``elementClass`` - the class to use for HTML Elements
* ``commentClass`` - the class to use for comments
* ``doctypeClass`` - the class to use for doctypes
It also has one required method:
* ``getDocument`` - Returns the root node of the complete document tree
3. If you wish to run the unit tests, you must also create a ``testSerializer``
method on your treebuilder which accepts a node and returns a string
containing Node and its children serialized according to the format used in
the unittests
"""
from __future__ import absolute_import, division, unicode_literals
from .._utils import default_etree
treeBuilderCache = {}
def getTreeBuilder(treeType, implementation=None, **kwargs):
"""Get a TreeBuilder class for various types of trees with built-in support
:arg treeType: the name of the tree type required (case-insensitive). Supported
values are:
* "dom" - A generic builder for DOM implementations, defaulting to a
xml.dom.minidom based implementation.
* "etree" - A generic builder for tree implementations exposing an
ElementTree-like interface, defaulting to xml.etree.cElementTree if
available and xml.etree.ElementTree if not.
* "lxml" - A etree-based builder for lxml.etree, handling limitations
of lxml's implementation.
:arg implementation: (Currently applies to the "etree" and "dom" tree
types). A module implementing the tree type e.g. xml.etree.ElementTree
or xml.etree.cElementTree.
:arg kwargs: Any additional options to pass to the TreeBuilder when
creating it.
Example:
>>> from html5lib.treebuilders import getTreeBuilder
>>> builder = getTreeBuilder('etree')
"""
treeType = treeType.lower()
if treeType not in treeBuilderCache:
if treeType == "dom":
from . import dom
# Come up with a sane default (pref. from the stdlib)
if implementation is None:
from xml.dom import minidom
implementation = minidom
# NEVER cache here, caching is done in the dom submodule
return dom.getDomModule(implementation, **kwargs).TreeBuilder
elif treeType == "lxml":
from . import etree_lxml
treeBuilderCache[treeType] = etree_lxml.TreeBuilder
elif treeType == "etree":
from . import etree
if implementation is None:
implementation = default_etree
# NEVER cache here, caching is done in the etree submodule
return etree.getETreeModule(implementation, **kwargs).TreeBuilder
else:
raise ValueError("""Unrecognised treebuilder "%s" """ % treeType)
return treeBuilderCache.get(treeType)
@@ -0,0 +1,417 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
from ..constants import scopingElements, tableInsertModeElements, namespaces
# The scope markers are inserted when entering object elements,
# marquees, table cells, and table captions, and are used to prevent formatting
# from "leaking" into tables, object elements, and marquees.
Marker = None
listElementsMap = {
None: (frozenset(scopingElements), False),
"button": (frozenset(scopingElements | {(namespaces["html"], "button")}), False),
"list": (frozenset(scopingElements | {(namespaces["html"], "ol"),
(namespaces["html"], "ul")}), False),
"table": (frozenset([(namespaces["html"], "html"),
(namespaces["html"], "table")]), False),
"select": (frozenset([(namespaces["html"], "optgroup"),
(namespaces["html"], "option")]), True)
}
class Node(object):
"""Represents an item in the tree"""
def __init__(self, name):
"""Creates a Node
:arg name: The tag name associated with the node
"""
# The tag name associated with the node
self.name = name
# The parent of the current node (or None for the document node)
self.parent = None
# The value of the current node (applies to text nodes and comments)
self.value = None
# A dict holding name -> value pairs for attributes of the node
self.attributes = {}
# A list of child nodes of the current node. This must include all
# elements but not necessarily other node types.
self.childNodes = []
# A list of miscellaneous flags that can be set on the node.
self._flags = []
def __str__(self):
attributesStr = " ".join(["%s=\"%s\"" % (name, value)
for name, value in
self.attributes.items()])
if attributesStr:
return "<%s %s>" % (self.name, attributesStr)
else:
return "<%s>" % (self.name)
def __repr__(self):
return "<%s>" % (self.name)
def appendChild(self, node):
"""Insert node as a child of the current node
:arg node: the node to insert
"""
raise NotImplementedError
def insertText(self, data, insertBefore=None):
"""Insert data as text in the current node, positioned before the
start of node insertBefore or to the end of the node's text.
:arg data: the data to insert
:arg insertBefore: True if you want to insert the text before the node
and False if you want to insert it after the node
"""
raise NotImplementedError
def insertBefore(self, node, refNode):
"""Insert node as a child of the current node, before refNode in the
list of child nodes. Raises ValueError if refNode is not a child of
the current node
:arg node: the node to insert
:arg refNode: the child node to insert the node before
"""
raise NotImplementedError
def removeChild(self, node):
"""Remove node from the children of the current node
:arg node: the child node to remove
"""
raise NotImplementedError
def reparentChildren(self, newParent):
"""Move all the children of the current node to newParent.
This is needed so that trees that don't store text as nodes move the
text in the correct way
:arg newParent: the node to move all this node's children to
"""
# XXX - should this method be made more general?
for child in self.childNodes:
newParent.appendChild(child)
self.childNodes = []
def cloneNode(self):
"""Return a shallow copy of the current node i.e. a node with the same
name and attributes but with no parent or child nodes
"""
raise NotImplementedError
def hasContent(self):
"""Return true if the node has children or text, false otherwise
"""
raise NotImplementedError
class ActiveFormattingElements(list):
def append(self, node):
equalCount = 0
if node != Marker:
for element in self[::-1]:
if element == Marker:
break
if self.nodesEqual(element, node):
equalCount += 1
if equalCount == 3:
self.remove(element)
break
list.append(self, node)
def nodesEqual(self, node1, node2):
if not node1.nameTuple == node2.nameTuple:
return False
if not node1.attributes == node2.attributes:
return False
return True
class TreeBuilder(object):
"""Base treebuilder implementation
* documentClass - the class to use for the bottommost node of a document
* elementClass - the class to use for HTML Elements
* commentClass - the class to use for comments
* doctypeClass - the class to use for doctypes
"""
# pylint:disable=not-callable
# Document class
documentClass = None
# The class to use for creating a node
elementClass = None
# The class to use for creating comments
commentClass = None
# The class to use for creating doctypes
doctypeClass = None
# Fragment class
fragmentClass = None
def __init__(self, namespaceHTMLElements):
"""Create a TreeBuilder
:arg namespaceHTMLElements: whether or not to namespace HTML elements
"""
if namespaceHTMLElements:
self.defaultNamespace = "http://www.w3.org/1999/xhtml"
else:
self.defaultNamespace = None
self.reset()
def reset(self):
self.openElements = []
self.activeFormattingElements = ActiveFormattingElements()
# XXX - rename these to headElement, formElement
self.headPointer = None
self.formPointer = None
self.insertFromTable = False
self.document = self.documentClass()
def elementInScope(self, target, variant=None):
# If we pass a node in we match that. if we pass a string
# match any node with that name
exactNode = hasattr(target, "nameTuple")
if not exactNode:
if isinstance(target, text_type):
target = (namespaces["html"], target)
assert isinstance(target, tuple)
listElements, invert = listElementsMap[variant]
for node in reversed(self.openElements):
if exactNode and node == target:
return True
elif not exactNode and node.nameTuple == target:
return True
elif (invert ^ (node.nameTuple in listElements)):
return False
assert False # We should never reach this point
def reconstructActiveFormattingElements(self):
# Within this algorithm the order of steps described in the
# specification is not quite the same as the order of steps in the
# code. It should still do the same though.
# Step 1: stop the algorithm when there's nothing to do.
if not self.activeFormattingElements:
return
# Step 2 and step 3: we start with the last element. So i is -1.
i = len(self.activeFormattingElements) - 1
entry = self.activeFormattingElements[i]
if entry == Marker or entry in self.openElements:
return
# Step 6
while entry != Marker and entry not in self.openElements:
if i == 0:
# This will be reset to 0 below
i = -1
break
i -= 1
# Step 5: let entry be one earlier in the list.
entry = self.activeFormattingElements[i]
while True:
# Step 7
i += 1
# Step 8
entry = self.activeFormattingElements[i]
clone = entry.cloneNode() # Mainly to get a new copy of the attributes
# Step 9
element = self.insertElement({"type": "StartTag",
"name": clone.name,
"namespace": clone.namespace,
"data": clone.attributes})
# Step 10
self.activeFormattingElements[i] = element
# Step 11
if element == self.activeFormattingElements[-1]:
break
def clearActiveFormattingElements(self):
entry = self.activeFormattingElements.pop()
while self.activeFormattingElements and entry != Marker:
entry = self.activeFormattingElements.pop()
def elementInActiveFormattingElements(self, name):
"""Check if an element exists between the end of the active
formatting elements and the last marker. If it does, return it, else
return false"""
for item in self.activeFormattingElements[::-1]:
# Check for Marker first because if it's a Marker it doesn't have a
# name attribute.
if item == Marker:
break
elif item.name == name:
return item
return False
def insertRoot(self, token):
element = self.createElement(token)
self.openElements.append(element)
self.document.appendChild(element)
def insertDoctype(self, token):
name = token["name"]
publicId = token["publicId"]
systemId = token["systemId"]
doctype = self.doctypeClass(name, publicId, systemId)
self.document.appendChild(doctype)
def insertComment(self, token, parent=None):
if parent is None:
parent = self.openElements[-1]
parent.appendChild(self.commentClass(token["data"]))
def createElement(self, token):
"""Create an element but don't insert it anywhere"""
name = token["name"]
namespace = token.get("namespace", self.defaultNamespace)
element = self.elementClass(name, namespace)
element.attributes = token["data"]
return element
def _getInsertFromTable(self):
return self._insertFromTable
def _setInsertFromTable(self, value):
"""Switch the function used to insert an element from the
normal one to the misnested table one and back again"""
self._insertFromTable = value
if value:
self.insertElement = self.insertElementTable
else:
self.insertElement = self.insertElementNormal
insertFromTable = property(_getInsertFromTable, _setInsertFromTable)
def insertElementNormal(self, token):
name = token["name"]
assert isinstance(name, text_type), "Element %s not unicode" % name
namespace = token.get("namespace", self.defaultNamespace)
element = self.elementClass(name, namespace)
element.attributes = token["data"]
self.openElements[-1].appendChild(element)
self.openElements.append(element)
return element
def insertElementTable(self, token):
"""Create an element and insert it into the tree"""
element = self.createElement(token)
if self.openElements[-1].name not in tableInsertModeElements:
return self.insertElementNormal(token)
else:
# We should be in the InTable mode. This means we want to do
# special magic element rearranging
parent, insertBefore = self.getTableMisnestedNodePosition()
if insertBefore is None:
parent.appendChild(element)
else:
parent.insertBefore(element, insertBefore)
self.openElements.append(element)
return element
def insertText(self, data, parent=None):
"""Insert text data."""
if parent is None:
parent = self.openElements[-1]
if (not self.insertFromTable or (self.insertFromTable and
self.openElements[-1].name
not in tableInsertModeElements)):
parent.insertText(data)
else:
# We should be in the InTable mode. This means we want to do
# special magic element rearranging
parent, insertBefore = self.getTableMisnestedNodePosition()
parent.insertText(data, insertBefore)
def getTableMisnestedNodePosition(self):
"""Get the foster parent element, and sibling to insert before
(or None) when inserting a misnested table node"""
# The foster parent element is the one which comes before the most
# recently opened table element
# XXX - this is really inelegant
lastTable = None
fosterParent = None
insertBefore = None
for elm in self.openElements[::-1]:
if elm.name == "table":
lastTable = elm
break
if lastTable:
# XXX - we should really check that this parent is actually a
# node here
if lastTable.parent:
fosterParent = lastTable.parent
insertBefore = lastTable
else:
fosterParent = self.openElements[
self.openElements.index(lastTable) - 1]
else:
fosterParent = self.openElements[0]
return fosterParent, insertBefore
def generateImpliedEndTags(self, exclude=None):
name = self.openElements[-1].name
# XXX td, th and tr are not actually needed
if (name in frozenset(("dd", "dt", "li", "option", "optgroup", "p", "rp", "rt")) and
name != exclude):
self.openElements.pop()
# XXX This is not entirely what the specification says. We should
# investigate it more closely.
self.generateImpliedEndTags(exclude)
def getDocument(self):
"""Return the final tree"""
return self.document
def getFragment(self):
"""Return the final fragment"""
# assert self.innerHTML
fragment = self.fragmentClass()
self.openElements[0].reparentChildren(fragment)
return fragment
def testSerializer(self, node):
"""Serialize the subtree of node in the format required by unit tests
:arg node: the node from which to start serializing
"""
raise NotImplementedError
@@ -0,0 +1,239 @@
from __future__ import absolute_import, division, unicode_literals
try:
from collections.abc import MutableMapping
except ImportError: # Python 2.7
from collections import MutableMapping
from xml.dom import minidom, Node
import weakref
from . import base
from .. import constants
from ..constants import namespaces
from .._utils import moduleFactoryFactory
def getDomBuilder(DomImplementation):
Dom = DomImplementation
class AttrList(MutableMapping):
def __init__(self, element):
self.element = element
def __iter__(self):
return iter(self.element.attributes.keys())
def __setitem__(self, name, value):
if isinstance(name, tuple):
raise NotImplementedError
else:
attr = self.element.ownerDocument.createAttribute(name)
attr.value = value
self.element.attributes[name] = attr
def __len__(self):
return len(self.element.attributes)
def items(self):
return list(self.element.attributes.items())
def values(self):
return list(self.element.attributes.values())
def __getitem__(self, name):
if isinstance(name, tuple):
raise NotImplementedError
else:
return self.element.attributes[name].value
def __delitem__(self, name):
if isinstance(name, tuple):
raise NotImplementedError
else:
del self.element.attributes[name]
class NodeBuilder(base.Node):
def __init__(self, element):
base.Node.__init__(self, element.nodeName)
self.element = element
namespace = property(lambda self: hasattr(self.element, "namespaceURI") and
self.element.namespaceURI or None)
def appendChild(self, node):
node.parent = self
self.element.appendChild(node.element)
def insertText(self, data, insertBefore=None):
text = self.element.ownerDocument.createTextNode(data)
if insertBefore:
self.element.insertBefore(text, insertBefore.element)
else:
self.element.appendChild(text)
def insertBefore(self, node, refNode):
self.element.insertBefore(node.element, refNode.element)
node.parent = self
def removeChild(self, node):
if node.element.parentNode == self.element:
self.element.removeChild(node.element)
node.parent = None
def reparentChildren(self, newParent):
while self.element.hasChildNodes():
child = self.element.firstChild
self.element.removeChild(child)
newParent.element.appendChild(child)
self.childNodes = []
def getAttributes(self):
return AttrList(self.element)
def setAttributes(self, attributes):
if attributes:
for name, value in list(attributes.items()):
if isinstance(name, tuple):
if name[0] is not None:
qualifiedName = (name[0] + ":" + name[1])
else:
qualifiedName = name[1]
self.element.setAttributeNS(name[2], qualifiedName,
value)
else:
self.element.setAttribute(
name, value)
attributes = property(getAttributes, setAttributes)
def cloneNode(self):
return NodeBuilder(self.element.cloneNode(False))
def hasContent(self):
return self.element.hasChildNodes()
def getNameTuple(self):
if self.namespace is None:
return namespaces["html"], self.name
else:
return self.namespace, self.name
nameTuple = property(getNameTuple)
class TreeBuilder(base.TreeBuilder): # pylint:disable=unused-variable
def documentClass(self):
self.dom = Dom.getDOMImplementation().createDocument(None, None, None)
return weakref.proxy(self)
def insertDoctype(self, token):
name = token["name"]
publicId = token["publicId"]
systemId = token["systemId"]
domimpl = Dom.getDOMImplementation()
doctype = domimpl.createDocumentType(name, publicId, systemId)
self.document.appendChild(NodeBuilder(doctype))
if Dom == minidom:
doctype.ownerDocument = self.dom
def elementClass(self, name, namespace=None):
if namespace is None and self.defaultNamespace is None:
node = self.dom.createElement(name)
else:
node = self.dom.createElementNS(namespace, name)
return NodeBuilder(node)
def commentClass(self, data):
return NodeBuilder(self.dom.createComment(data))
def fragmentClass(self):
return NodeBuilder(self.dom.createDocumentFragment())
def appendChild(self, node):
self.dom.appendChild(node.element)
def testSerializer(self, element):
return testSerializer(element)
def getDocument(self):
return self.dom
def getFragment(self):
return base.TreeBuilder.getFragment(self).element
def insertText(self, data, parent=None):
data = data
if parent != self:
base.TreeBuilder.insertText(self, data, parent)
else:
# HACK: allow text nodes as children of the document node
if hasattr(self.dom, '_child_node_types'):
# pylint:disable=protected-access
if Node.TEXT_NODE not in self.dom._child_node_types:
self.dom._child_node_types = list(self.dom._child_node_types)
self.dom._child_node_types.append(Node.TEXT_NODE)
self.dom.appendChild(self.dom.createTextNode(data))
implementation = DomImplementation
name = None
def testSerializer(element):
element.normalize()
rv = []
def serializeElement(element, indent=0):
if element.nodeType == Node.DOCUMENT_TYPE_NODE:
if element.name:
if element.publicId or element.systemId:
publicId = element.publicId or ""
systemId = element.systemId or ""
rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
(' ' * indent, element.name, publicId, systemId))
else:
rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, element.name))
else:
rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
elif element.nodeType == Node.DOCUMENT_NODE:
rv.append("#document")
elif element.nodeType == Node.DOCUMENT_FRAGMENT_NODE:
rv.append("#document-fragment")
elif element.nodeType == Node.COMMENT_NODE:
rv.append("|%s<!-- %s -->" % (' ' * indent, element.nodeValue))
elif element.nodeType == Node.TEXT_NODE:
rv.append("|%s\"%s\"" % (' ' * indent, element.nodeValue))
else:
if (hasattr(element, "namespaceURI") and
element.namespaceURI is not None):
name = "%s %s" % (constants.prefixes[element.namespaceURI],
element.nodeName)
else:
name = element.nodeName
rv.append("|%s<%s>" % (' ' * indent, name))
if element.hasAttributes():
attributes = []
for i in range(len(element.attributes)):
attr = element.attributes.item(i)
name = attr.nodeName
value = attr.value
ns = attr.namespaceURI
if ns:
name = "%s %s" % (constants.prefixes[ns], attr.localName)
else:
name = attr.nodeName
attributes.append((name, value))
for name, value in sorted(attributes):
rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
indent += 2
for child in element.childNodes:
serializeElement(child, indent)
serializeElement(element, 0)
return "\n".join(rv)
return locals()
# The actual means to get a module!
getDomModule = moduleFactoryFactory(getDomBuilder)
@@ -0,0 +1,343 @@
from __future__ import absolute_import, division, unicode_literals
# pylint:disable=protected-access
from tensorboard._vendor.bleach.six_shim import text_type
import re
from copy import copy
from . import base
from .. import _ihatexml
from .. import constants
from ..constants import namespaces
from .._utils import moduleFactoryFactory
tag_regexp = re.compile("{([^}]*)}(.*)")
def getETreeBuilder(ElementTreeImplementation, fullTree=False):
ElementTree = ElementTreeImplementation
ElementTreeCommentType = ElementTree.Comment("asd").tag
class Element(base.Node):
def __init__(self, name, namespace=None):
self._name = name
self._namespace = namespace
self._element = ElementTree.Element(self._getETreeTag(name,
namespace))
if namespace is None:
self.nameTuple = namespaces["html"], self._name
else:
self.nameTuple = self._namespace, self._name
self.parent = None
self._childNodes = []
self._flags = []
def _getETreeTag(self, name, namespace):
if namespace is None:
etree_tag = name
else:
etree_tag = "{%s}%s" % (namespace, name)
return etree_tag
def _setName(self, name):
self._name = name
self._element.tag = self._getETreeTag(self._name, self._namespace)
def _getName(self):
return self._name
name = property(_getName, _setName)
def _setNamespace(self, namespace):
self._namespace = namespace
self._element.tag = self._getETreeTag(self._name, self._namespace)
def _getNamespace(self):
return self._namespace
namespace = property(_getNamespace, _setNamespace)
def _getAttributes(self):
return self._element.attrib
def _setAttributes(self, attributes):
el_attrib = self._element.attrib
el_attrib.clear()
if attributes:
# calling .items _always_ allocates, and the above truthy check is cheaper than the
# allocation on average
for key, value in attributes.items():
if isinstance(key, tuple):
name = "{%s}%s" % (key[2], key[1])
else:
name = key
el_attrib[name] = value
attributes = property(_getAttributes, _setAttributes)
def _getChildNodes(self):
return self._childNodes
def _setChildNodes(self, value):
del self._element[:]
self._childNodes = []
for element in value:
self.insertChild(element)
childNodes = property(_getChildNodes, _setChildNodes)
def hasContent(self):
"""Return true if the node has children or text"""
return bool(self._element.text or len(self._element))
def appendChild(self, node):
self._childNodes.append(node)
self._element.append(node._element)
node.parent = self
def insertBefore(self, node, refNode):
index = list(self._element).index(refNode._element)
self._element.insert(index, node._element)
node.parent = self
def removeChild(self, node):
self._childNodes.remove(node)
self._element.remove(node._element)
node.parent = None
def insertText(self, data, insertBefore=None):
if not(len(self._element)):
if not self._element.text:
self._element.text = ""
self._element.text += data
elif insertBefore is None:
# Insert the text as the tail of the last child element
if not self._element[-1].tail:
self._element[-1].tail = ""
self._element[-1].tail += data
else:
# Insert the text before the specified node
children = list(self._element)
index = children.index(insertBefore._element)
if index > 0:
if not self._element[index - 1].tail:
self._element[index - 1].tail = ""
self._element[index - 1].tail += data
else:
if not self._element.text:
self._element.text = ""
self._element.text += data
def cloneNode(self):
element = type(self)(self.name, self.namespace)
if self._element.attrib:
element._element.attrib = copy(self._element.attrib)
return element
def reparentChildren(self, newParent):
if newParent.childNodes:
newParent.childNodes[-1]._element.tail += self._element.text
else:
if not newParent._element.text:
newParent._element.text = ""
if self._element.text is not None:
newParent._element.text += self._element.text
self._element.text = ""
base.Node.reparentChildren(self, newParent)
class Comment(Element):
def __init__(self, data):
# Use the superclass constructor to set all properties on the
# wrapper element
self._element = ElementTree.Comment(data)
self.parent = None
self._childNodes = []
self._flags = []
def _getData(self):
return self._element.text
def _setData(self, value):
self._element.text = value
data = property(_getData, _setData)
class DocumentType(Element):
def __init__(self, name, publicId, systemId):
Element.__init__(self, "<!DOCTYPE>")
self._element.text = name
self.publicId = publicId
self.systemId = systemId
def _getPublicId(self):
return self._element.get("publicId", "")
def _setPublicId(self, value):
if value is not None:
self._element.set("publicId", value)
publicId = property(_getPublicId, _setPublicId)
def _getSystemId(self):
return self._element.get("systemId", "")
def _setSystemId(self, value):
if value is not None:
self._element.set("systemId", value)
systemId = property(_getSystemId, _setSystemId)
class Document(Element):
def __init__(self):
Element.__init__(self, "DOCUMENT_ROOT")
class DocumentFragment(Element):
def __init__(self):
Element.__init__(self, "DOCUMENT_FRAGMENT")
def testSerializer(element):
rv = []
def serializeElement(element, indent=0):
if not(hasattr(element, "tag")):
element = element.getroot()
if element.tag == "<!DOCTYPE>":
if element.get("publicId") or element.get("systemId"):
publicId = element.get("publicId") or ""
systemId = element.get("systemId") or ""
rv.append("""<!DOCTYPE %s "%s" "%s">""" %
(element.text, publicId, systemId))
else:
rv.append("<!DOCTYPE %s>" % (element.text,))
elif element.tag == "DOCUMENT_ROOT":
rv.append("#document")
if element.text is not None:
rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text))
if element.tail is not None:
raise TypeError("Document node cannot have tail")
if hasattr(element, "attrib") and len(element.attrib):
raise TypeError("Document node cannot have attributes")
elif element.tag == ElementTreeCommentType:
rv.append("|%s<!-- %s -->" % (' ' * indent, element.text))
else:
assert isinstance(element.tag, text_type), \
"Expected unicode, got %s, %s" % (type(element.tag), element.tag)
nsmatch = tag_regexp.match(element.tag)
if nsmatch is None:
name = element.tag
else:
ns, name = nsmatch.groups()
prefix = constants.prefixes[ns]
name = "%s %s" % (prefix, name)
rv.append("|%s<%s>" % (' ' * indent, name))
if hasattr(element, "attrib"):
attributes = []
for name, value in element.attrib.items():
nsmatch = tag_regexp.match(name)
if nsmatch is not None:
ns, name = nsmatch.groups()
prefix = constants.prefixes[ns]
attr_string = "%s %s" % (prefix, name)
else:
attr_string = name
attributes.append((attr_string, value))
for name, value in sorted(attributes):
rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
if element.text:
rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text))
indent += 2
for child in element:
serializeElement(child, indent)
if element.tail:
rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail))
serializeElement(element, 0)
return "\n".join(rv)
def tostring(element): # pylint:disable=unused-variable
"""Serialize an element and its child nodes to a string"""
rv = []
filter = _ihatexml.InfosetFilter()
def serializeElement(element):
if isinstance(element, ElementTree.ElementTree):
element = element.getroot()
if element.tag == "<!DOCTYPE>":
if element.get("publicId") or element.get("systemId"):
publicId = element.get("publicId") or ""
systemId = element.get("systemId") or ""
rv.append("""<!DOCTYPE %s PUBLIC "%s" "%s">""" %
(element.text, publicId, systemId))
else:
rv.append("<!DOCTYPE %s>" % (element.text,))
elif element.tag == "DOCUMENT_ROOT":
if element.text is not None:
rv.append(element.text)
if element.tail is not None:
raise TypeError("Document node cannot have tail")
if hasattr(element, "attrib") and len(element.attrib):
raise TypeError("Document node cannot have attributes")
for child in element:
serializeElement(child)
elif element.tag == ElementTreeCommentType:
rv.append("<!--%s-->" % (element.text,))
else:
# This is assumed to be an ordinary element
if not element.attrib:
rv.append("<%s>" % (filter.fromXmlName(element.tag),))
else:
attr = " ".join(["%s=\"%s\"" % (
filter.fromXmlName(name), value)
for name, value in element.attrib.items()])
rv.append("<%s %s>" % (element.tag, attr))
if element.text:
rv.append(element.text)
for child in element:
serializeElement(child)
rv.append("</%s>" % (element.tag,))
if element.tail:
rv.append(element.tail)
serializeElement(element)
return "".join(rv)
class TreeBuilder(base.TreeBuilder): # pylint:disable=unused-variable
documentClass = Document
doctypeClass = DocumentType
elementClass = Element
commentClass = Comment
fragmentClass = DocumentFragment
implementation = ElementTreeImplementation
def testSerializer(self, element):
return testSerializer(element)
def getDocument(self):
if fullTree:
return self.document._element
else:
if self.defaultNamespace is not None:
return self.document._element.find(
"{%s}html" % self.defaultNamespace)
else:
return self.document._element.find("html")
def getFragment(self):
return base.TreeBuilder.getFragment(self)._element
return locals()
getETreeModule = moduleFactoryFactory(getETreeBuilder)
@@ -0,0 +1,392 @@
"""Module for supporting the lxml.etree library. The idea here is to use as much
of the native library as possible, without using fragile hacks like custom element
names that break between releases. The downside of this is that we cannot represent
all possible trees; specifically the following are known to cause problems:
Text or comments as siblings of the root element
Docypes with no name
When any of these things occur, we emit a DataLossWarning
"""
from __future__ import absolute_import, division, unicode_literals
# pylint:disable=protected-access
import warnings
import re
import sys
try:
from collections.abc import MutableMapping
except ImportError:
from collections import MutableMapping
from . import base
from ..constants import DataLossWarning
from .. import constants
from . import etree as etree_builders
from .. import _ihatexml
import lxml.etree as etree
from tensorboard._vendor.bleach.six_shim import PY3, binary_type
fullTree = True
tag_regexp = re.compile("{([^}]*)}(.*)")
comment_type = etree.Comment("asd").tag
class DocumentType(object):
def __init__(self, name, publicId, systemId):
self.name = name
self.publicId = publicId
self.systemId = systemId
class Document(object):
def __init__(self):
self._elementTree = None
self._childNodes = []
def appendChild(self, element):
last = self._elementTree.getroot()
for last in self._elementTree.getroot().itersiblings():
pass
last.addnext(element._element)
def _getChildNodes(self):
return self._childNodes
childNodes = property(_getChildNodes)
def testSerializer(element):
rv = []
infosetFilter = _ihatexml.InfosetFilter(preventDoubleDashComments=True)
def serializeElement(element, indent=0):
if not hasattr(element, "tag"):
if hasattr(element, "getroot"):
# Full tree case
rv.append("#document")
if element.docinfo.internalDTD:
if not (element.docinfo.public_id or
element.docinfo.system_url):
dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name
else:
dtd_str = """<!DOCTYPE %s "%s" "%s">""" % (
element.docinfo.root_name,
element.docinfo.public_id,
element.docinfo.system_url)
rv.append("|%s%s" % (' ' * (indent + 2), dtd_str))
next_element = element.getroot()
while next_element.getprevious() is not None:
next_element = next_element.getprevious()
while next_element is not None:
serializeElement(next_element, indent + 2)
next_element = next_element.getnext()
elif isinstance(element, str) or isinstance(element, bytes):
# Text in a fragment
assert isinstance(element, str) or sys.version_info[0] == 2
rv.append("|%s\"%s\"" % (' ' * indent, element))
else:
# Fragment case
rv.append("#document-fragment")
for next_element in element:
serializeElement(next_element, indent + 2)
elif element.tag == comment_type:
rv.append("|%s<!-- %s -->" % (' ' * indent, element.text))
if hasattr(element, "tail") and element.tail:
rv.append("|%s\"%s\"" % (' ' * indent, element.tail))
else:
assert isinstance(element, etree._Element)
nsmatch = etree_builders.tag_regexp.match(element.tag)
if nsmatch is not None:
ns = nsmatch.group(1)
tag = nsmatch.group(2)
prefix = constants.prefixes[ns]
rv.append("|%s<%s %s>" % (' ' * indent, prefix,
infosetFilter.fromXmlName(tag)))
else:
rv.append("|%s<%s>" % (' ' * indent,
infosetFilter.fromXmlName(element.tag)))
if hasattr(element, "attrib"):
attributes = []
for name, value in element.attrib.items():
nsmatch = tag_regexp.match(name)
if nsmatch is not None:
ns, name = nsmatch.groups()
name = infosetFilter.fromXmlName(name)
prefix = constants.prefixes[ns]
attr_string = "%s %s" % (prefix, name)
else:
attr_string = infosetFilter.fromXmlName(name)
attributes.append((attr_string, value))
for name, value in sorted(attributes):
rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
if element.text:
rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text))
indent += 2
for child in element:
serializeElement(child, indent)
if hasattr(element, "tail") and element.tail:
rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail))
serializeElement(element, 0)
return "\n".join(rv)
def tostring(element):
"""Serialize an element and its child nodes to a string"""
rv = []
def serializeElement(element):
if not hasattr(element, "tag"):
if element.docinfo.internalDTD:
if element.docinfo.doctype:
dtd_str = element.docinfo.doctype
else:
dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name
rv.append(dtd_str)
serializeElement(element.getroot())
elif element.tag == comment_type:
rv.append("<!--%s-->" % (element.text,))
else:
# This is assumed to be an ordinary element
if not element.attrib:
rv.append("<%s>" % (element.tag,))
else:
attr = " ".join(["%s=\"%s\"" % (name, value)
for name, value in element.attrib.items()])
rv.append("<%s %s>" % (element.tag, attr))
if element.text:
rv.append(element.text)
for child in element:
serializeElement(child)
rv.append("</%s>" % (element.tag,))
if hasattr(element, "tail") and element.tail:
rv.append(element.tail)
serializeElement(element)
return "".join(rv)
class TreeBuilder(base.TreeBuilder):
documentClass = Document
doctypeClass = DocumentType
elementClass = None
commentClass = None
fragmentClass = Document
implementation = etree
def __init__(self, namespaceHTMLElements, fullTree=False):
builder = etree_builders.getETreeModule(etree, fullTree=fullTree)
infosetFilter = self.infosetFilter = _ihatexml.InfosetFilter(preventDoubleDashComments=True)
self.namespaceHTMLElements = namespaceHTMLElements
class Attributes(MutableMapping):
def __init__(self, element):
self._element = element
def _coerceKey(self, key):
if isinstance(key, tuple):
name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1]))
else:
name = infosetFilter.coerceAttribute(key)
return name
def __getitem__(self, key):
value = self._element._element.attrib[self._coerceKey(key)]
if not PY3 and isinstance(value, binary_type):
value = value.decode("ascii")
return value
def __setitem__(self, key, value):
self._element._element.attrib[self._coerceKey(key)] = value
def __delitem__(self, key):
del self._element._element.attrib[self._coerceKey(key)]
def __iter__(self):
return iter(self._element._element.attrib)
def __len__(self):
return len(self._element._element.attrib)
def clear(self):
return self._element._element.attrib.clear()
class Element(builder.Element):
def __init__(self, name, namespace):
name = infosetFilter.coerceElement(name)
builder.Element.__init__(self, name, namespace=namespace)
self._attributes = Attributes(self)
def _setName(self, name):
self._name = infosetFilter.coerceElement(name)
self._element.tag = self._getETreeTag(
self._name, self._namespace)
def _getName(self):
return infosetFilter.fromXmlName(self._name)
name = property(_getName, _setName)
def _getAttributes(self):
return self._attributes
def _setAttributes(self, value):
attributes = self.attributes
attributes.clear()
attributes.update(value)
attributes = property(_getAttributes, _setAttributes)
def insertText(self, data, insertBefore=None):
data = infosetFilter.coerceCharacters(data)
builder.Element.insertText(self, data, insertBefore)
def cloneNode(self):
element = type(self)(self.name, self.namespace)
if self._element.attrib:
element._element.attrib.update(self._element.attrib)
return element
class Comment(builder.Comment):
def __init__(self, data):
data = infosetFilter.coerceComment(data)
builder.Comment.__init__(self, data)
def _setData(self, data):
data = infosetFilter.coerceComment(data)
self._element.text = data
def _getData(self):
return self._element.text
data = property(_getData, _setData)
self.elementClass = Element
self.commentClass = Comment
# self.fragmentClass = builder.DocumentFragment
base.TreeBuilder.__init__(self, namespaceHTMLElements)
def reset(self):
base.TreeBuilder.reset(self)
self.insertComment = self.insertCommentInitial
self.initial_comments = []
self.doctype = None
def testSerializer(self, element):
return testSerializer(element)
def getDocument(self):
if fullTree:
return self.document._elementTree
else:
return self.document._elementTree.getroot()
def getFragment(self):
fragment = []
element = self.openElements[0]._element
if element.text:
fragment.append(element.text)
fragment.extend(list(element))
if element.tail:
fragment.append(element.tail)
return fragment
def insertDoctype(self, token):
name = token["name"]
publicId = token["publicId"]
systemId = token["systemId"]
if not name:
warnings.warn("lxml cannot represent empty doctype", DataLossWarning)
self.doctype = None
else:
coercedName = self.infosetFilter.coerceElement(name)
if coercedName != name:
warnings.warn("lxml cannot represent non-xml doctype", DataLossWarning)
doctype = self.doctypeClass(coercedName, publicId, systemId)
self.doctype = doctype
def insertCommentInitial(self, data, parent=None):
assert parent is None or parent is self.document
assert self.document._elementTree is None
self.initial_comments.append(data)
def insertCommentMain(self, data, parent=None):
if (parent == self.document and
self.document._elementTree.getroot()[-1].tag == comment_type):
warnings.warn("lxml cannot represent adjacent comments beyond the root elements", DataLossWarning)
super(TreeBuilder, self).insertComment(data, parent)
def insertRoot(self, token):
# Because of the way libxml2 works, it doesn't seem to be possible to
# alter information like the doctype after the tree has been parsed.
# Therefore we need to use the built-in parser to create our initial
# tree, after which we can add elements like normal
docStr = ""
if self.doctype:
assert self.doctype.name
docStr += "<!DOCTYPE %s" % self.doctype.name
if (self.doctype.publicId is not None or
self.doctype.systemId is not None):
docStr += (' PUBLIC "%s" ' %
(self.infosetFilter.coercePubid(self.doctype.publicId or "")))
if self.doctype.systemId:
sysid = self.doctype.systemId
if sysid.find("'") >= 0 and sysid.find('"') >= 0:
warnings.warn("DOCTYPE system cannot contain single and double quotes", DataLossWarning)
sysid = sysid.replace("'", 'U00027')
if sysid.find("'") >= 0:
docStr += '"%s"' % sysid
else:
docStr += "'%s'" % sysid
else:
docStr += "''"
docStr += ">"
if self.doctype.name != token["name"]:
warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning)
docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>"
root = etree.fromstring(docStr)
# Append the initial comments:
for comment_token in self.initial_comments:
comment = self.commentClass(comment_token["data"])
root.addprevious(comment._element)
# Create the root document and add the ElementTree to it
self.document = self.documentClass()
self.document._elementTree = root.getroottree()
# Give the root element the right name
name = token["name"]
namespace = token.get("namespace", self.defaultNamespace)
if namespace is None:
etree_tag = name
else:
etree_tag = "{%s}%s" % (namespace, name)
root.tag = etree_tag
# Add the root element to the internal child/open data structures
root_element = self.elementClass(name, namespace)
root_element._element = root
self.document._childNodes.append(root_element)
self.openElements.append(root_element)
# Reset to the default insert comment function
self.insertComment = self.insertCommentMain
@@ -0,0 +1,154 @@
"""A collection of modules for iterating through different kinds of
tree, generating tokens identical to those produced by the tokenizer
module.
To create a tree walker for a new type of tree, you need to
implement a tree walker object (called TreeWalker by convention) that
implements a 'serialize' method which takes a tree as sole argument and
returns an iterator which generates tokens.
"""
from __future__ import absolute_import, division, unicode_literals
from .. import constants
from .._utils import default_etree
__all__ = ["getTreeWalker", "pprint"]
treeWalkerCache = {}
def getTreeWalker(treeType, implementation=None, **kwargs):
"""Get a TreeWalker class for various types of tree with built-in support
:arg str treeType: the name of the tree type required (case-insensitive).
Supported values are:
* "dom": The xml.dom.minidom DOM implementation
* "etree": A generic walker for tree implementations exposing an
elementtree-like interface (known to work with ElementTree,
cElementTree and lxml.etree).
* "lxml": Optimized walker for lxml.etree
* "genshi": a Genshi stream
:arg implementation: A module implementing the tree type e.g.
xml.etree.ElementTree or cElementTree (Currently applies to the "etree"
tree type only).
:arg kwargs: keyword arguments passed to the etree walker--for other
walkers, this has no effect
:returns: a TreeWalker class
"""
treeType = treeType.lower()
if treeType not in treeWalkerCache:
if treeType == "dom":
from . import dom
treeWalkerCache[treeType] = dom.TreeWalker
elif treeType == "genshi":
from . import genshi
treeWalkerCache[treeType] = genshi.TreeWalker
elif treeType == "lxml":
from . import etree_lxml
treeWalkerCache[treeType] = etree_lxml.TreeWalker
elif treeType == "etree":
from . import etree
if implementation is None:
implementation = default_etree
# XXX: NEVER cache here, caching is done in the etree submodule
return etree.getETreeModule(implementation, **kwargs).TreeWalker
return treeWalkerCache.get(treeType)
def concatenateCharacterTokens(tokens):
pendingCharacters = []
for token in tokens:
type = token["type"]
if type in ("Characters", "SpaceCharacters"):
pendingCharacters.append(token["data"])
else:
if pendingCharacters:
yield {"type": "Characters", "data": "".join(pendingCharacters)}
pendingCharacters = []
yield token
if pendingCharacters:
yield {"type": "Characters", "data": "".join(pendingCharacters)}
def pprint(walker):
"""Pretty printer for tree walkers
Takes a TreeWalker instance and pretty prints the output of walking the tree.
:arg walker: a TreeWalker instance
"""
output = []
indent = 0
for token in concatenateCharacterTokens(walker):
type = token["type"]
if type in ("StartTag", "EmptyTag"):
# tag name
if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
if token["namespace"] in constants.prefixes:
ns = constants.prefixes[token["namespace"]]
else:
ns = token["namespace"]
name = "%s %s" % (ns, token["name"])
else:
name = token["name"]
output.append("%s<%s>" % (" " * indent, name))
indent += 2
# attributes (sorted for consistent ordering)
attrs = token["data"]
for (namespace, localname), value in sorted(attrs.items()):
if namespace:
if namespace in constants.prefixes:
ns = constants.prefixes[namespace]
else:
ns = namespace
name = "%s %s" % (ns, localname)
else:
name = localname
output.append("%s%s=\"%s\"" % (" " * indent, name, value))
# self-closing
if type == "EmptyTag":
indent -= 2
elif type == "EndTag":
indent -= 2
elif type == "Comment":
output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
elif type == "Doctype":
if token["name"]:
if token["publicId"]:
output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
(" " * indent,
token["name"],
token["publicId"],
token["systemId"] if token["systemId"] else ""))
elif token["systemId"]:
output.append("""%s<!DOCTYPE %s "" "%s">""" %
(" " * indent,
token["name"],
token["systemId"]))
else:
output.append("%s<!DOCTYPE %s>" % (" " * indent,
token["name"]))
else:
output.append("%s<!DOCTYPE >" % (" " * indent,))
elif type == "Characters":
output.append("%s\"%s\"" % (" " * indent, token["data"]))
elif type == "SpaceCharacters":
assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
else:
raise ValueError("Unknown token type, %s" % type)
return "\n".join(output)
@@ -0,0 +1,252 @@
from __future__ import absolute_import, division, unicode_literals
from xml.dom import Node
from ..constants import namespaces, voidElements, spaceCharacters
__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
"TreeWalker", "NonRecursiveTreeWalker"]
DOCUMENT = Node.DOCUMENT_NODE
DOCTYPE = Node.DOCUMENT_TYPE_NODE
TEXT = Node.TEXT_NODE
ELEMENT = Node.ELEMENT_NODE
COMMENT = Node.COMMENT_NODE
ENTITY = Node.ENTITY_NODE
UNKNOWN = "<#UNKNOWN#>"
spaceCharacters = "".join(spaceCharacters)
class TreeWalker(object):
"""Walks a tree yielding tokens
Tokens are dicts that all have a ``type`` field specifying the type of the
token.
"""
def __init__(self, tree):
"""Creates a TreeWalker
:arg tree: the tree to walk
"""
self.tree = tree
def __iter__(self):
raise NotImplementedError
def error(self, msg):
"""Generates an error token with the given message
:arg msg: the error message
:returns: SerializeError token
"""
return {"type": "SerializeError", "data": msg}
def emptyTag(self, namespace, name, attrs, hasChildren=False):
"""Generates an EmptyTag token
:arg namespace: the namespace of the token--can be ``None``
:arg name: the name of the element
:arg attrs: the attributes of the element as a dict
:arg hasChildren: whether or not to yield a SerializationError because
this tag shouldn't have children
:returns: EmptyTag token
"""
yield {"type": "EmptyTag", "name": name,
"namespace": namespace,
"data": attrs}
if hasChildren:
yield self.error("Void element has children")
def startTag(self, namespace, name, attrs):
"""Generates a StartTag token
:arg namespace: the namespace of the token--can be ``None``
:arg name: the name of the element
:arg attrs: the attributes of the element as a dict
:returns: StartTag token
"""
return {"type": "StartTag",
"name": name,
"namespace": namespace,
"data": attrs}
def endTag(self, namespace, name):
"""Generates an EndTag token
:arg namespace: the namespace of the token--can be ``None``
:arg name: the name of the element
:returns: EndTag token
"""
return {"type": "EndTag",
"name": name,
"namespace": namespace}
def text(self, data):
"""Generates SpaceCharacters and Characters tokens
Depending on what's in the data, this generates one or more
``SpaceCharacters`` and ``Characters`` tokens.
For example:
>>> from html5lib.treewalkers.base import TreeWalker
>>> # Give it an empty tree just so it instantiates
>>> walker = TreeWalker([])
>>> list(walker.text(''))
[]
>>> list(walker.text(' '))
[{u'data': ' ', u'type': u'SpaceCharacters'}]
>>> list(walker.text(' abc ')) # doctest: +NORMALIZE_WHITESPACE
[{u'data': ' ', u'type': u'SpaceCharacters'},
{u'data': u'abc', u'type': u'Characters'},
{u'data': u' ', u'type': u'SpaceCharacters'}]
:arg data: the text data
:returns: one or more ``SpaceCharacters`` and ``Characters`` tokens
"""
data = data
middle = data.lstrip(spaceCharacters)
left = data[:len(data) - len(middle)]
if left:
yield {"type": "SpaceCharacters", "data": left}
data = middle
middle = data.rstrip(spaceCharacters)
right = data[len(middle):]
if middle:
yield {"type": "Characters", "data": middle}
if right:
yield {"type": "SpaceCharacters", "data": right}
def comment(self, data):
"""Generates a Comment token
:arg data: the comment
:returns: Comment token
"""
return {"type": "Comment", "data": data}
def doctype(self, name, publicId=None, systemId=None):
"""Generates a Doctype token
:arg name:
:arg publicId:
:arg systemId:
:returns: the Doctype token
"""
return {"type": "Doctype",
"name": name,
"publicId": publicId,
"systemId": systemId}
def entity(self, name):
"""Generates an Entity token
:arg name: the entity name
:returns: an Entity token
"""
return {"type": "Entity", "name": name}
def unknown(self, nodeType):
"""Handles unknown node types"""
return self.error("Unknown node type: " + nodeType)
class NonRecursiveTreeWalker(TreeWalker):
def getNodeDetails(self, node):
raise NotImplementedError
def getFirstChild(self, node):
raise NotImplementedError
def getNextSibling(self, node):
raise NotImplementedError
def getParentNode(self, node):
raise NotImplementedError
def __iter__(self):
currentNode = self.tree
while currentNode is not None:
details = self.getNodeDetails(currentNode)
type, details = details[0], details[1:]
hasChildren = False
if type == DOCTYPE:
yield self.doctype(*details)
elif type == TEXT:
for token in self.text(*details):
yield token
elif type == ELEMENT:
namespace, name, attributes, hasChildren = details
if (not namespace or namespace == namespaces["html"]) and name in voidElements:
for token in self.emptyTag(namespace, name, attributes,
hasChildren):
yield token
hasChildren = False
else:
yield self.startTag(namespace, name, attributes)
elif type == COMMENT:
yield self.comment(details[0])
elif type == ENTITY:
yield self.entity(details[0])
elif type == DOCUMENT:
hasChildren = True
else:
yield self.unknown(details[0])
if hasChildren:
firstChild = self.getFirstChild(currentNode)
else:
firstChild = None
if firstChild is not None:
currentNode = firstChild
else:
while currentNode is not None:
details = self.getNodeDetails(currentNode)
type, details = details[0], details[1:]
if type == ELEMENT:
namespace, name, attributes, hasChildren = details
if (namespace and namespace != namespaces["html"]) or name not in voidElements:
yield self.endTag(namespace, name)
if self.tree is currentNode:
currentNode = None
break
nextSibling = self.getNextSibling(currentNode)
if nextSibling is not None:
currentNode = nextSibling
break
else:
currentNode = self.getParentNode(currentNode)
@@ -0,0 +1,43 @@
from __future__ import absolute_import, division, unicode_literals
from xml.dom import Node
from . import base
class TreeWalker(base.NonRecursiveTreeWalker):
def getNodeDetails(self, node):
if node.nodeType == Node.DOCUMENT_TYPE_NODE:
return base.DOCTYPE, node.name, node.publicId, node.systemId
elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
return base.TEXT, node.nodeValue
elif node.nodeType == Node.ELEMENT_NODE:
attrs = {}
for attr in list(node.attributes.keys()):
attr = node.getAttributeNode(attr)
if attr.namespaceURI:
attrs[(attr.namespaceURI, attr.localName)] = attr.value
else:
attrs[(None, attr.name)] = attr.value
return (base.ELEMENT, node.namespaceURI, node.nodeName,
attrs, node.hasChildNodes())
elif node.nodeType == Node.COMMENT_NODE:
return base.COMMENT, node.nodeValue
elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
return (base.DOCUMENT,)
else:
return base.UNKNOWN, node.nodeType
def getFirstChild(self, node):
return node.firstChild
def getNextSibling(self, node):
return node.nextSibling
def getParentNode(self, node):
return node.parentNode
@@ -0,0 +1,131 @@
from __future__ import absolute_import, division, unicode_literals
from collections import OrderedDict
import re
from tensorboard._vendor.bleach.six_shim import string_types
from . import base
from .._utils import moduleFactoryFactory
tag_regexp = re.compile("{([^}]*)}(.*)")
def getETreeBuilder(ElementTreeImplementation):
ElementTree = ElementTreeImplementation
ElementTreeCommentType = ElementTree.Comment("asd").tag
class TreeWalker(base.NonRecursiveTreeWalker): # pylint:disable=unused-variable
"""Given the particular ElementTree representation, this implementation,
to avoid using recursion, returns "nodes" as tuples with the following
content:
1. The current element
2. The index of the element relative to its parent
3. A stack of ancestor elements
4. A flag "text", "tail" or None to indicate if the current node is a
text node; either the text or tail of the current element (1)
"""
def getNodeDetails(self, node):
if isinstance(node, tuple): # It might be the root Element
elt, _, _, flag = node
if flag in ("text", "tail"):
return base.TEXT, getattr(elt, flag)
else:
node = elt
if not(hasattr(node, "tag")):
node = node.getroot()
if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"):
return (base.DOCUMENT,)
elif node.tag == "<!DOCTYPE>":
return (base.DOCTYPE, node.text,
node.get("publicId"), node.get("systemId"))
elif node.tag == ElementTreeCommentType:
return base.COMMENT, node.text
else:
assert isinstance(node.tag, string_types), type(node.tag)
# This is assumed to be an ordinary element
match = tag_regexp.match(node.tag)
if match:
namespace, tag = match.groups()
else:
namespace = None
tag = node.tag
attrs = OrderedDict()
for name, value in list(node.attrib.items()):
match = tag_regexp.match(name)
if match:
attrs[(match.group(1), match.group(2))] = value
else:
attrs[(None, name)] = value
return (base.ELEMENT, namespace, tag,
attrs, len(node) or node.text)
def getFirstChild(self, node):
if isinstance(node, tuple):
element, key, parents, flag = node
else:
element, key, parents, flag = node, None, [], None
if flag in ("text", "tail"):
return None
else:
if element.text:
return element, key, parents, "text"
elif len(element):
parents.append(element)
return element[0], 0, parents, None
else:
return None
def getNextSibling(self, node):
if isinstance(node, tuple):
element, key, parents, flag = node
else:
return None
if flag == "text":
if len(element):
parents.append(element)
return element[0], 0, parents, None
else:
return None
else:
if element.tail and flag != "tail":
return element, key, parents, "tail"
elif key < len(parents[-1]) - 1:
return parents[-1][key + 1], key + 1, parents, None
else:
return None
def getParentNode(self, node):
if isinstance(node, tuple):
element, key, parents, flag = node
else:
return None
if flag == "text":
if not parents:
return element
else:
return element, key, parents, None
else:
parent = parents.pop()
if not parents:
return parent
else:
assert list(parents[-1]).count(parent) == 1
return parent, list(parents[-1]).index(parent), parents, None
return locals()
getETreeModule = moduleFactoryFactory(getETreeBuilder)
@@ -0,0 +1,215 @@
from __future__ import absolute_import, division, unicode_literals
from tensorboard._vendor.bleach.six_shim import text_type
from collections import OrderedDict
from lxml import etree
from ..treebuilders.etree import tag_regexp
from . import base
from .. import _ihatexml
def ensure_str(s):
if s is None:
return None
elif isinstance(s, text_type):
return s
else:
return s.decode("ascii", "strict")
class Root(object):
def __init__(self, et):
self.elementtree = et
self.children = []
try:
if et.docinfo.internalDTD:
self.children.append(Doctype(self,
ensure_str(et.docinfo.root_name),
ensure_str(et.docinfo.public_id),
ensure_str(et.docinfo.system_url)))
except AttributeError:
pass
try:
node = et.getroot()
except AttributeError:
node = et
while node.getprevious() is not None:
node = node.getprevious()
while node is not None:
self.children.append(node)
node = node.getnext()
self.text = None
self.tail = None
def __getitem__(self, key):
return self.children[key]
def getnext(self):
return None
def __len__(self):
return 1
class Doctype(object):
def __init__(self, root_node, name, public_id, system_id):
self.root_node = root_node
self.name = name
self.public_id = public_id
self.system_id = system_id
self.text = None
self.tail = None
def getnext(self):
return self.root_node.children[1]
class FragmentRoot(Root):
def __init__(self, children):
self.children = [FragmentWrapper(self, child) for child in children]
self.text = self.tail = None
def getnext(self):
return None
class FragmentWrapper(object):
def __init__(self, fragment_root, obj):
self.root_node = fragment_root
self.obj = obj
if hasattr(self.obj, 'text'):
self.text = ensure_str(self.obj.text)
else:
self.text = None
if hasattr(self.obj, 'tail'):
self.tail = ensure_str(self.obj.tail)
else:
self.tail = None
def __getattr__(self, name):
return getattr(self.obj, name)
def getnext(self):
siblings = self.root_node.children
idx = siblings.index(self)
if idx < len(siblings) - 1:
return siblings[idx + 1]
else:
return None
def __getitem__(self, key):
return self.obj[key]
def __bool__(self):
return bool(self.obj)
def getparent(self):
return None
def __str__(self):
return str(self.obj)
def __unicode__(self):
return str(self.obj)
def __len__(self):
return len(self.obj)
class TreeWalker(base.NonRecursiveTreeWalker):
def __init__(self, tree):
# pylint:disable=redefined-variable-type
if isinstance(tree, list):
self.fragmentChildren = set(tree)
tree = FragmentRoot(tree)
else:
self.fragmentChildren = set()
tree = Root(tree)
base.NonRecursiveTreeWalker.__init__(self, tree)
self.filter = _ihatexml.InfosetFilter()
def getNodeDetails(self, node):
if isinstance(node, tuple): # Text node
node, key = node
assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
return base.TEXT, ensure_str(getattr(node, key))
elif isinstance(node, Root):
return (base.DOCUMENT,)
elif isinstance(node, Doctype):
return base.DOCTYPE, node.name, node.public_id, node.system_id
elif isinstance(node, FragmentWrapper) and not hasattr(node, "tag"):
return base.TEXT, ensure_str(node.obj)
elif node.tag == etree.Comment:
return base.COMMENT, ensure_str(node.text)
elif node.tag == etree.Entity:
return base.ENTITY, ensure_str(node.text)[1:-1] # strip &;
else:
# This is assumed to be an ordinary element
match = tag_regexp.match(ensure_str(node.tag))
if match:
namespace, tag = match.groups()
else:
namespace = None
tag = ensure_str(node.tag)
attrs = OrderedDict()
for name, value in list(node.attrib.items()):
name = ensure_str(name)
value = ensure_str(value)
match = tag_regexp.match(name)
if match:
attrs[(match.group(1), match.group(2))] = value
else:
attrs[(None, name)] = value
return (base.ELEMENT, namespace, self.filter.fromXmlName(tag),
attrs, len(node) > 0 or node.text)
def getFirstChild(self, node):
assert not isinstance(node, tuple), "Text nodes have no children"
assert len(node) or node.text, "Node has no children"
if node.text:
return (node, "text")
else:
return node[0]
def getNextSibling(self, node):
if isinstance(node, tuple): # Text node
node, key = node
assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
if key == "text":
# XXX: we cannot use a "bool(node) and node[0] or None" construct here
# because node[0] might evaluate to False if it has no child element
if len(node):
return node[0]
else:
return None
else: # tail
return node.getnext()
return (node, "tail") if node.tail else node.getnext()
def getParentNode(self, node):
if isinstance(node, tuple): # Text node
node, key = node
assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
if key == "text":
return node
# else: fallback to "normal" processing
elif node in self.fragmentChildren:
return None
return node.getparent()
@@ -0,0 +1,69 @@
from __future__ import absolute_import, division, unicode_literals
from genshi.core import QName
from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT
from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
from . import base
from ..constants import voidElements, namespaces
class TreeWalker(base.TreeWalker):
def __iter__(self):
# Buffer the events so we can pass in the following one
previous = None
for event in self.tree:
if previous is not None:
for token in self.tokens(previous, event):
yield token
previous = event
# Don't forget the final event!
if previous is not None:
for token in self.tokens(previous, None):
yield token
def tokens(self, event, next):
kind, data, _ = event
if kind == START:
tag, attribs = data
name = tag.localname
namespace = tag.namespace
converted_attribs = {}
for k, v in attribs:
if isinstance(k, QName):
converted_attribs[(k.namespace, k.localname)] = v
else:
converted_attribs[(None, k)] = v
if namespace == namespaces["html"] and name in voidElements:
for token in self.emptyTag(namespace, name, converted_attribs,
not next or next[0] != END or
next[1] != tag):
yield token
else:
yield self.startTag(namespace, name, converted_attribs)
elif kind == END:
name = data.localname
namespace = data.namespace
if namespace != namespaces["html"] or name not in voidElements:
yield self.endTag(namespace, name)
elif kind == COMMENT:
yield self.comment(data)
elif kind == TEXT:
for token in self.text(data):
yield token
elif kind == DOCTYPE:
yield self.doctype(*data)
elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS,
START_CDATA, END_CDATA, PI):
pass
else:
yield self.unknown(kind)
@@ -0,0 +1,32 @@
"""A set of basic callbacks for bleach.linkify."""
def nofollow(attrs, new=False):
href_key = (None, "href")
if href_key not in attrs:
return attrs
if attrs[href_key].startswith("mailto:"):
return attrs
rel_key = (None, "rel")
rel_values = [val for val in attrs.get(rel_key, "").split(" ") if val]
if "nofollow" not in [rel_val.lower() for rel_val in rel_values]:
rel_values.append("nofollow")
attrs[rel_key] = " ".join(rel_values)
return attrs
def target_blank(attrs, new=False):
href_key = (None, "href")
if href_key not in attrs:
return attrs
if attrs[href_key].startswith("mailto:"):
return attrs
attrs[(None, "target")] = "_blank"
return attrs
@@ -0,0 +1,757 @@
# flake8: noqa
"""
Shim module between Bleach and html5lib. This makes it easier to upgrade the
html5lib library without having to change a lot of code.
"""
import re
import string
import warnings
# ignore html5lib deprecation warnings to use bleach; we are bleach
# apply before we import submodules that import html5lib
warnings.filterwarnings(
"ignore",
message="html5lib's sanitizer is deprecated",
category=DeprecationWarning,
module="bleach._vendor.html5lib",
)
from tensorboard._vendor.bleach._vendor.html5lib import ( # noqa: E402 module level import not at top of file
HTMLParser,
getTreeWalker,
)
from tensorboard._vendor.bleach._vendor.html5lib import (
constants,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib.constants import ( # noqa: E402 module level import not at top of file
namespaces,
prefixes,
)
from tensorboard._vendor.bleach._vendor.html5lib.constants import (
_ReparseException as ReparseException,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib.filters.base import (
Filter,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib.filters.sanitizer import (
allowed_protocols,
allowed_css_properties,
allowed_svg_properties,
attr_val_is_uri,
svg_attr_val_allows_ref,
svg_allow_local_href,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib.filters.sanitizer import (
Filter as SanitizerFilter,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib._inputstream import (
HTMLInputStream,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib.serializer import (
escape,
HTMLSerializer,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib._tokenizer import (
attributeMap,
HTMLTokenizer,
) # noqa: E402 module level import not at top of file
from tensorboard._vendor.bleach._vendor.html5lib._trie import (
Trie,
) # noqa: E402 module level import not at top of file
#: Map of entity name to expanded entity
ENTITIES = constants.entities
#: Trie of html entity string -> character representation
ENTITIES_TRIE = Trie(ENTITIES)
#: Token type constants--these never change
TAG_TOKEN_TYPES = {
constants.tokenTypes["StartTag"],
constants.tokenTypes["EndTag"],
constants.tokenTypes["EmptyTag"],
}
TAG_TOKEN_TYPE_START = constants.tokenTypes["StartTag"]
TAG_TOKEN_TYPE_END = constants.tokenTypes["EndTag"]
TAG_TOKEN_TYPE_CHARACTERS = constants.tokenTypes["Characters"]
TAG_TOKEN_TYPE_PARSEERROR = constants.tokenTypes["ParseError"]
#: List of valid HTML tags, from WHATWG HTML Living Standard as of 2018-10-17
#: https://html.spec.whatwg.org/multipage/indices.html#elements-3
HTML_TAGS = frozenset(
(
"a",
"abbr",
"address",
"area",
"article",
"aside",
"audio",
"b",
"base",
"bdi",
"bdo",
"blockquote",
"body",
"br",
"button",
"canvas",
"caption",
"cite",
"code",
"col",
"colgroup",
"data",
"datalist",
"dd",
"del",
"details",
"dfn",
"dialog",
"div",
"dl",
"dt",
"em",
"embed",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"head",
"header",
"hgroup",
"hr",
"html",
"i",
"iframe",
"img",
"input",
"ins",
"kbd",
"keygen",
"label",
"legend",
"li",
"link",
"map",
"mark",
"menu",
"meta",
"meter",
"nav",
"noscript",
"object",
"ol",
"optgroup",
"option",
"output",
"p",
"param",
"picture",
"pre",
"progress",
"q",
"rp",
"rt",
"ruby",
"s",
"samp",
"script",
"section",
"select",
"slot",
"small",
"source",
"span",
"strong",
"style",
"sub",
"summary",
"sup",
"table",
"tbody",
"td",
"template",
"textarea",
"tfoot",
"th",
"thead",
"time",
"title",
"tr",
"track",
"u",
"ul",
"var",
"video",
"wbr",
)
)
#: List of block level HTML tags, as per https://github.com/mozilla/bleach/issues/369
#: from mozilla on 2019.07.11
#: https://developer.mozilla.org/en-US/docs/Web/HTML/Block-level_elements#Elements
HTML_TAGS_BLOCK_LEVEL = frozenset(
(
"address",
"article",
"aside",
"blockquote",
"details",
"dialog",
"dd",
"div",
"dl",
"dt",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hgroup",
"hr",
"li",
"main",
"nav",
"ol",
"p",
"pre",
"section",
"table",
"ul",
)
)
class InputStreamWithMemory:
"""Wraps an HTMLInputStream to remember characters since last <
This wraps existing HTMLInputStream classes to keep track of the stream
since the last < which marked an open tag state.
"""
def __init__(self, inner_stream):
self._inner_stream = inner_stream
self.reset = self._inner_stream.reset
self.position = self._inner_stream.position
self._buffer = []
@property
def errors(self):
return self._inner_stream.errors
@property
def charEncoding(self):
return self._inner_stream.charEncoding
@property
def changeEncoding(self):
return self._inner_stream.changeEncoding
def char(self):
c = self._inner_stream.char()
# char() can return None if EOF, so ignore that
if c:
self._buffer.append(c)
return c
def charsUntil(self, characters, opposite=False):
chars = self._inner_stream.charsUntil(characters, opposite=opposite)
self._buffer.extend(list(chars))
return chars
def unget(self, char):
if self._buffer:
self._buffer.pop(-1)
return self._inner_stream.unget(char)
def get_tag(self):
"""Returns the stream history since last '<'
Since the buffer starts at the last '<' as as seen by tagOpenState(),
we know that everything from that point to when this method is called
is the "tag" that is being tokenized.
"""
return "".join(self._buffer)
def start_tag(self):
"""Resets stream history to just '<'
This gets called by tagOpenState() which marks a '<' that denotes an
open tag. Any time we see that, we reset the buffer.
"""
self._buffer = ["<"]
class BleachHTMLTokenizer(HTMLTokenizer):
"""Tokenizer that doesn't consume character entities"""
def __init__(self, consume_entities=False, **kwargs):
super().__init__(**kwargs)
self.consume_entities = consume_entities
# Wrap the stream with one that remembers the history
self.stream = InputStreamWithMemory(self.stream)
# Remember the last token emitted; needed for block element spacing
self.emitted_last_token = None
def __iter__(self):
last_error_token = None
for token in super().__iter__():
if last_error_token is not None:
if (
last_error_token["data"] == "invalid-character-in-attribute-name"
and token["type"] in TAG_TOKEN_TYPES
and token.get("data")
):
# token["data"] is an html5lib attributeMap
# (OrderedDict 3.7+ and dict otherwise)
# of attr name to attr value
#
# Remove attribute names that have ', " or < in them
# because those characters are invalid for attribute names.
token["data"] = attributeMap(
(attr_name, attr_value)
for attr_name, attr_value in token["data"].items()
if (
'"' not in attr_name
and "'" not in attr_name
and "<" not in attr_name
)
)
last_error_token = None
yield token
elif (
last_error_token["data"] == "expected-closing-tag-but-got-char"
and self.parser.tags is not None
and token["data"].lower().strip() not in self.parser.tags
):
# We've got either a malformed tag or a pseudo-tag or
# something that html5lib wants to turn into a malformed
# comment which Bleach clean() will drop so we interfere
# with the token stream to handle it more correctly.
#
# If this is an allowed tag, it's malformed and we just let
# the html5lib parser deal with it--we don't enter into this
# block.
#
# If this is not an allowed tag, then we convert it to
# characters and it'll get escaped in the sanitizer.
token["data"] = self.stream.get_tag()
token["type"] = TAG_TOKEN_TYPE_CHARACTERS
last_error_token = None
yield token
elif token["type"] == TAG_TOKEN_TYPE_PARSEERROR:
# If the token is a parse error, then let the last_error_token
# go, and make token the new last_error_token
yield last_error_token
last_error_token = token
else:
yield last_error_token
yield token
last_error_token = None
continue
# If the token is a ParseError, we hold on to it so we can get the
# next token and potentially fix it.
if token["type"] == TAG_TOKEN_TYPE_PARSEERROR:
last_error_token = token
continue
yield token
if last_error_token:
if last_error_token["data"] == "eof-in-tag-name":
# Handle the case where the text being parsed ends with <
# followed by a series of characters. It's treated as a tag
# name that abruptly ends, but we should treat that like
# character data
yield {"type": TAG_TOKEN_TYPE_CHARACTERS, "data": self.stream.get_tag()}
elif last_error_token["data"] in (
"duplicate-attribute",
"eof-in-attribute-name",
"eof-in-attribute-value-no-quotes",
"expected-end-of-tag-but-got-eof",
):
# Handle the case where the text being parsed ends with <
# followed by characters and then space and then:
#
# * more characters
# * more characters repeated with a space between (e.g. "abc abc")
# * more characters and then a space and then an EOF (e.g. "abc def ")
#
# These cases are treated as a tag name followed by an
# attribute that abruptly ends, but we should treat that like
# character data instead.
yield {"type": TAG_TOKEN_TYPE_CHARACTERS, "data": self.stream.get_tag()}
else:
yield last_error_token
def consumeEntity(self, allowedChar=None, fromAttribute=False):
# If this tokenizer is set to consume entities, then we can let the
# superclass do its thing.
if self.consume_entities:
return super().consumeEntity(allowedChar, fromAttribute)
# If this tokenizer is set to not consume entities, then we don't want
# to consume and convert them, so this overrides the html5lib tokenizer's
# consumeEntity so that it's now a no-op.
#
# However, when that gets called, it's consumed an &, so we put that back in
# the stream.
if fromAttribute:
self.currentToken["data"][-1][1] += "&"
else:
self.tokenQueue.append({"type": TAG_TOKEN_TYPE_CHARACTERS, "data": "&"})
def tagOpenState(self):
# This state marks a < that is either a StartTag, EndTag, EmptyTag,
# or ParseError. In all cases, we want to drop any stream history
# we've collected so far and we do that by calling start_tag() on
# the input stream wrapper.
self.stream.start_tag()
return super().tagOpenState()
def emitCurrentToken(self):
token = self.currentToken
if (
self.parser.tags is not None
and token["type"] in TAG_TOKEN_TYPES
and token["name"].lower() not in self.parser.tags
):
# If this is a start/end/empty tag for a tag that's not in our
# allowed list, then it gets stripped or escaped. In both of these
# cases it gets converted to a Characters token.
if self.parser.strip:
if (
self.emitted_last_token
and token["type"] == TAG_TOKEN_TYPE_START
and token["name"].lower() in HTML_TAGS_BLOCK_LEVEL
):
# If this is a block level tag we're stripping, we drop it
# for a newline because that's what a browser would parse
# it as
new_data = "\n"
else:
# For all other things being stripped, we throw in an empty
# string token
new_data = ""
else:
# If we're escaping the token, we want to escape the exact
# original string. Since tokenizing also normalizes data
# and this is a tag-like thing, we've lost some information.
# So we go back through the stream to get the original
# string and use that.
new_data = self.stream.get_tag()
new_token = {"type": TAG_TOKEN_TYPE_CHARACTERS, "data": new_data}
self.currentToken = self.emitted_last_token = new_token
self.tokenQueue.append(new_token)
self.state = self.dataState
return
self.emitted_last_token = self.currentToken
super().emitCurrentToken()
class BleachHTMLParser(HTMLParser):
"""Parser that uses BleachHTMLTokenizer"""
def __init__(self, tags, strip, consume_entities, **kwargs):
"""
:arg tags: set of allowed tags--everything else is either stripped or
escaped; if None, then this doesn't look at tags at all
:arg strip: whether to strip disallowed tags (True) or escape them (False);
if tags=None, then this doesn't have any effect
:arg consume_entities: whether to consume entities (default behavior) or
leave them as is when tokenizing (BleachHTMLTokenizer-added behavior)
"""
self.tags = (
frozenset((tag.lower() for tag in tags)) if tags is not None else None
)
self.strip = strip
self.consume_entities = consume_entities
super().__init__(**kwargs)
def _parse(
self, stream, innerHTML=False, container="div", scripting=True, **kwargs
):
# set scripting=True to parse <noscript> as though JS is enabled to
# match the expected context in browsers
#
# https://html.spec.whatwg.org/multipage/scripting.html#the-noscript-element
#
# Override HTMLParser so we can swap out the tokenizer for our own.
self.innerHTMLMode = innerHTML
self.container = container
self.scripting = scripting
self.tokenizer = BleachHTMLTokenizer(
stream=stream, consume_entities=self.consume_entities, parser=self, **kwargs
)
self.reset()
try:
self.mainLoop()
except ReparseException:
self.reset()
self.mainLoop()
def convert_entity(value):
"""Convert an entity (minus the & and ; part) into what it represents
This handles numeric, hex, and text entities.
:arg value: the string (minus the ``&`` and ``;`` part) to convert
:returns: unicode character or None if it's an ambiguous ampersand that
doesn't match a character entity
"""
if value[0] == "#":
if len(value) < 2:
return None
if value[1] in ("x", "X"):
# hex-encoded code point
int_as_string, base = value[2:], 16
else:
# decimal code point
int_as_string, base = value[1:], 10
if int_as_string == "":
return None
code_point = int(int_as_string, base)
if 0 < code_point < 0x110000:
return chr(code_point)
else:
return None
return ENTITIES.get(value, None)
def convert_entities(text):
"""Converts all found entities in the text
:arg text: the text to convert entities in
:returns: unicode text with converted entities
"""
if "&" not in text:
return text
new_text = []
for part in next_possible_entity(text):
if not part:
continue
if part.startswith("&"):
entity = match_entity(part)
if entity is not None:
converted = convert_entity(entity)
# If it's not an ambiguous ampersand, then replace with the
# unicode character. Otherwise, we leave the entity in.
if converted is not None:
new_text.append(converted)
remainder = part[len(entity) + 2 :]
if part:
new_text.append(remainder)
continue
new_text.append(part)
return "".join(new_text)
def match_entity(stream):
"""Returns first entity in stream or None if no entity exists
Note: For Bleach purposes, entities must start with a "&" and end with a
";". This ignores ambiguous character entities that have no ";" at the end.
:arg stream: the character stream
:returns: the entity string without "&" or ";" if it's a valid character
entity; ``None`` otherwise
"""
# Nix the & at the beginning
if stream[0] != "&":
raise ValueError('Stream should begin with "&"')
stream = stream[1:]
stream = list(stream)
possible_entity = ""
end_characters = "<&=;" + string.whitespace
# Handle number entities
if stream and stream[0] == "#":
possible_entity = "#"
stream.pop(0)
if stream and stream[0] in ("x", "X"):
allowed = "0123456789abcdefABCDEF"
possible_entity += stream.pop(0)
else:
allowed = "0123456789"
# FIXME(willkg): Do we want to make sure these are valid number
# entities? This doesn't do that currently.
while stream and stream[0] not in end_characters:
c = stream.pop(0)
if c not in allowed:
break
possible_entity += c
if possible_entity and stream and stream[0] == ";":
return possible_entity
return None
# Handle character entities
while stream and stream[0] not in end_characters:
c = stream.pop(0)
possible_entity += c
if not ENTITIES_TRIE.has_keys_with_prefix(possible_entity):
# If it's not a prefix, then it's not an entity and we're
# out
return None
if possible_entity and stream and stream[0] == ";":
return possible_entity
return None
AMP_SPLIT_RE = re.compile("(&)")
def next_possible_entity(text):
"""Takes a text and generates a list of possible entities
:arg text: the text to look at
:returns: generator where each part (except the first) starts with an
"&"
"""
for i, part in enumerate(AMP_SPLIT_RE.split(text)):
if i == 0:
yield part
elif i % 2 == 0:
yield "&" + part
class BleachHTMLSerializer(HTMLSerializer):
"""HTMLSerializer that undoes & -> &amp; in attributes and sets
escape_rcdata to True
"""
# per the HTMLSerializer.__init__ docstring:
#
# Whether to escape characters that need to be
# escaped within normal elements within rcdata elements such as
# style.
#
escape_rcdata = True
def escape_base_amp(self, stoken):
"""Escapes just bare & in HTML attribute values"""
# First, undo escaping of &. We need to do this because html5lib's
# HTMLSerializer expected the tokenizer to consume all the character
# entities and convert them to their respective characters, but the
# BleachHTMLTokenizer doesn't do that. For example, this fixes
# &amp;entity; back to &entity; .
stoken = stoken.replace("&amp;", "&")
# However, we do want all bare & that are not marking character
# entities to be changed to &amp;, so let's do that carefully here.
for part in next_possible_entity(stoken):
if not part:
continue
if part.startswith("&"):
entity = match_entity(part)
# Only leave entities in that are not ambiguous. If they're
# ambiguous, then we escape the ampersand.
if entity is not None and convert_entity(entity) is not None:
yield f"&{entity};"
# Length of the entity plus 2--one for & at the beginning
# and one for ; at the end
part = part[len(entity) + 2 :]
if part:
yield part
continue
yield part.replace("&", "&amp;")
def serialize(self, treewalker, encoding=None):
"""Wrap HTMLSerializer.serialize and conver & to &amp; in attribute values
Note that this converts & to &amp; in attribute values where the & isn't
already part of an unambiguous character entity.
"""
in_tag = False
after_equals = False
for stoken in super().serialize(treewalker, encoding):
if in_tag:
if stoken == ">":
in_tag = False
elif after_equals:
if stoken != '"':
yield from self.escape_base_amp(stoken)
after_equals = False
continue
elif stoken == "=":
after_equals = True
yield stoken
else:
if stoken.startswith("<"):
in_tag = True
yield stoken
@@ -0,0 +1,633 @@
import re
from urllib.parse import quote
from tensorboard._vendor.bleach import callbacks as linkify_callbacks
from tensorboard._vendor.bleach import html5lib_shim
#: List of default callbacks
DEFAULT_CALLBACKS = [linkify_callbacks.nofollow]
TLDS = """ac ad ae aero af ag ai al am an ao aq ar arpa as asia at au aw ax az
ba bb bd be bf bg bh bi biz bj bm bn bo br bs bt bv bw by bz ca cat
cc cd cf cg ch ci ck cl cm cn co com coop cr cu cv cx cy cz de dj dk
dm do dz ec edu ee eg er es et eu fi fj fk fm fo fr ga gb gd ge gf gg
gh gi gl gm gn gov gp gq gr gs gt gu gw gy hk hm hn hr ht hu id ie il
im in info int io iq ir is it je jm jo jobs jp ke kg kh ki km kn kp
kr kw ky kz la lb lc li lk lr ls lt lu lv ly ma mc md me mg mh mil mk
ml mm mn mo mobi mp mq mr ms mt mu museum mv mw mx my mz na name nc ne
net nf ng ni nl no np nr nu nz om org pa pe pf pg ph pk pl pm pn post
pr pro ps pt pw py qa re ro rs ru rw sa sb sc sd se sg sh si sj sk sl
sm sn so sr ss st su sv sx sy sz tc td tel tf tg th tj tk tl tm tn to
tp tr travel tt tv tw tz ua ug uk us uy uz va vc ve vg vi vn vu wf ws
xn xxx ye yt yu za zm zw""".split()
# Make sure that .com doesn't get matched by .co first
TLDS.reverse()
def build_url_re(tlds=TLDS, protocols=html5lib_shim.allowed_protocols):
"""Builds the url regex used by linkifier
If you want a different set of tlds or allowed protocols, pass those in
and stomp on the existing ``url_re``::
from bleach import linkifier
my_url_re = linkifier.build_url_re(my_tlds_list, my_protocols)
linker = LinkifyFilter(url_re=my_url_re)
"""
return re.compile(
r"""\(* # Match any opening parentheses.
\b(?<![@.])(?:(?:{0}):/{{0,3}}(?:(?:\w+:)?\w+@)?)? # http://
([\w-]+\.)+(?:{1})(?:\:[0-9]+)?(?!\.\w)\b # xx.yy.tld(:##)?
(?:[/?][^\s\{{\}}\|\\\^`<>"]*)?
# /path/zz (excluding "unsafe" chars from RFC 3986,
# except for # and ~, which happen in practice)
""".format(
"|".join(sorted(protocols)), "|".join(sorted(tlds))
),
re.IGNORECASE | re.VERBOSE | re.UNICODE,
)
URL_RE = build_url_re()
PROTO_RE = re.compile(r"^[\w-]+:/{0,3}", re.IGNORECASE)
def build_email_re(tlds=TLDS):
"""Builds the email regex used by linkifier
If you want a different set of tlds, pass those in and stomp on the existing ``email_re``::
from bleach import linkifier
my_email_re = linkifier.build_email_re(my_tlds_list)
linker = LinkifyFilter(email_re=my_url_re)
"""
# open and closing braces doubled below for format string
return re.compile(
r"""(?<!//)
(([-!#$%&'*+/=?^_`{{}}|~0-9A-Z]+
(\.[-!#$%&'*+/=?^_`{{}}|~0-9A-Z]+)* # dot-atom
|^"([\001-\010\013\014\016-\037!#-\[\]-\177]
|\\[\001-\011\013\014\016-\177])*" # quoted-string
)@(?:[A-Z0-9](?:[A-Z0-9-]{{0,61}}[A-Z0-9])?\.)+(?:{0})) # domain
""".format(
"|".join(tlds)
),
re.IGNORECASE | re.MULTILINE | re.VERBOSE,
)
EMAIL_RE = build_email_re()
class Linker:
"""Convert URL-like strings in an HTML fragment to links
This function converts strings that look like URLs, domain names and email
addresses in text that may be an HTML fragment to links, while preserving:
1. links already in the string
2. urls found in attributes
3. email addresses
linkify does a best-effort approach and tries to recover from bad
situations due to crazy text.
"""
def __init__(
self,
callbacks=DEFAULT_CALLBACKS,
skip_tags=None,
parse_email=False,
url_re=URL_RE,
email_re=EMAIL_RE,
recognized_tags=html5lib_shim.HTML_TAGS,
):
"""Creates a Linker instance
:arg list callbacks: list of callbacks to run when adjusting tag attributes;
defaults to ``bleach.linkifier.DEFAULT_CALLBACKS``
:arg set skip_tags: set of tags that you don't want to linkify the
contents of; for example, you could set this to ``{'pre'}`` to skip
linkifying contents of ``pre`` tags; ``None`` means you don't
want linkify to skip any tags
:arg bool parse_email: whether or not to linkify email addresses
:arg url_re: url matching regex
:arg email_re: email matching regex
:arg set recognized_tags: the set of tags that linkify knows about;
everything else gets escaped
:returns: linkified text as unicode
"""
self.callbacks = callbacks
self.skip_tags = skip_tags
self.parse_email = parse_email
self.url_re = url_re
self.email_re = email_re
# Create a parser/tokenizer that allows all HTML tags and escapes
# anything not in that list.
self.parser = html5lib_shim.BleachHTMLParser(
tags=frozenset(recognized_tags),
strip=False,
consume_entities=False,
namespaceHTMLElements=False,
)
self.walker = html5lib_shim.getTreeWalker("etree")
self.serializer = html5lib_shim.BleachHTMLSerializer(
quote_attr_values="always",
omit_optional_tags=False,
# We want to leave entities as they are without escaping or
# resolving or expanding
resolve_entities=False,
# linkify does not sanitize
sanitize=False,
# linkify preserves attr order
alphabetical_attributes=False,
)
def linkify(self, text):
"""Linkify specified text
:arg str text: the text to add links to
:returns: linkified text as unicode
:raises TypeError: if ``text`` is not a text type
"""
if not isinstance(text, str):
raise TypeError("argument must be of text type")
if not text:
return ""
dom = self.parser.parseFragment(text)
filtered = LinkifyFilter(
source=self.walker(dom),
callbacks=self.callbacks,
skip_tags=self.skip_tags,
parse_email=self.parse_email,
url_re=self.url_re,
email_re=self.email_re,
)
return self.serializer.render(filtered)
class LinkifyFilter(html5lib_shim.Filter):
"""html5lib filter that linkifies text
This will do the following:
* convert email addresses into links
* convert urls into links
* edit existing links by running them through callbacks--the default is to
add a ``rel="nofollow"``
This filter can be used anywhere html5lib filters can be used.
"""
def __init__(
self,
source,
callbacks=DEFAULT_CALLBACKS,
skip_tags=None,
parse_email=False,
url_re=URL_RE,
email_re=EMAIL_RE,
):
"""Creates a LinkifyFilter instance
:arg source: stream as an html5lib TreeWalker
:arg list callbacks: list of callbacks to run when adjusting tag attributes;
defaults to ``bleach.linkifier.DEFAULT_CALLBACKS``
:arg set skip_tags: set of tags that you don't want to linkify the
contents of; for example, you could set this to ``{'pre'}`` to skip
linkifying contents of ``pre`` tags
:arg bool parse_email: whether or not to linkify email addresses
:arg url_re: url matching regex
:arg email_re: email matching regex
"""
super().__init__(source)
self.callbacks = callbacks or []
self.skip_tags = skip_tags or {}
self.parse_email = parse_email
self.url_re = url_re
self.email_re = email_re
def apply_callbacks(self, attrs, is_new):
"""Given an attrs dict and an is_new bool, runs through callbacks
Callbacks can return an adjusted attrs dict or ``None``. In the case of
``None``, we stop going through callbacks and return that and the link
gets dropped.
:arg dict attrs: map of ``(namespace, name)`` -> ``value``
:arg bool is_new: whether or not this link was added by linkify
:returns: adjusted attrs dict or ``None``
"""
for cb in self.callbacks:
attrs = cb(attrs, is_new)
if attrs is None:
return None
return attrs
def extract_character_data(self, token_list):
"""Extracts and squashes character sequences in a token stream"""
# FIXME(willkg): This is a terrible idea. What it does is drop all the
# tags from the token list and merge the Characters and SpaceCharacters
# tokens into a single text.
#
# So something like this::
#
# "<span>" "<b>" "some text" "</b>" "</span>"
#
# gets converted to "some text".
#
# This gets used to figure out the ``_text`` fauxttribute value for
# linkify callables.
#
# I'm not really sure how else to support that ``_text`` fauxttribute and
# maintain some modicum of backwards compatibility with previous versions
# of Bleach.
out = []
for token in token_list:
token_type = token["type"]
if token_type in ["Characters", "SpaceCharacters"]:
out.append(token["data"])
return "".join(out)
def handle_email_addresses(self, src_iter):
"""Handle email addresses in character tokens"""
for token in src_iter:
if token["type"] == "Characters":
text = token["data"]
new_tokens = []
end = 0
# For each email address we find in the text
for match in self.email_re.finditer(text):
if match.start() > end:
new_tokens.append(
{"type": "Characters", "data": text[end : match.start()]}
)
# URL-encode the "local-part" according to RFC6068
parts = match.group(0).split("@")
parts[0] = quote(parts[0])
address = "@".join(parts)
# Run attributes through the callbacks to see what we
# should do with this match
attrs = {
(None, "href"): "mailto:%s" % address,
"_text": match.group(0),
}
attrs = self.apply_callbacks(attrs, True)
if attrs is None:
# Just add the text--but not as a link
new_tokens.append(
{"type": "Characters", "data": match.group(0)}
)
else:
# Add an "a" tag for the new link
_text = attrs.pop("_text", "")
new_tokens.extend(
[
{"type": "StartTag", "name": "a", "data": attrs},
{"type": "Characters", "data": str(_text)},
{"type": "EndTag", "name": "a"},
]
)
end = match.end()
if new_tokens:
# Yield the adjusted set of tokens and then continue
# through the loop
if end < len(text):
new_tokens.append({"type": "Characters", "data": text[end:]})
yield from new_tokens
continue
yield token
def strip_non_url_bits(self, fragment):
"""Strips non-url bits from the url
This accounts for over-eager matching by the regex.
"""
prefix = suffix = ""
while fragment:
# Try removing ( from the beginning and, if it's balanced, from the
# end, too
if fragment.startswith("("):
prefix = prefix + "("
fragment = fragment[1:]
if fragment.endswith(")"):
suffix = ")" + suffix
fragment = fragment[:-1]
continue
# Now try extraneous things from the end. For example, sometimes we
# pick up ) at the end of a url, but the url is in a parenthesized
# phrase like:
#
# "i looked at the site (at http://example.com)"
if fragment.endswith(")") and "(" not in fragment:
fragment = fragment[:-1]
suffix = ")" + suffix
continue
# Handle commas
if fragment.endswith(","):
fragment = fragment[:-1]
suffix = "," + suffix
continue
# Handle periods
if fragment.endswith("."):
fragment = fragment[:-1]
suffix = "." + suffix
continue
# Nothing matched, so we're done
break
return fragment, prefix, suffix
def handle_links(self, src_iter):
"""Handle links in character tokens"""
in_a = False # happens, if parse_email=True and if a mail was found
for token in src_iter:
if in_a:
if token["type"] == "EndTag" and token["name"] == "a":
in_a = False
yield token
continue
elif token["type"] == "StartTag" and token["name"] == "a":
in_a = True
yield token
continue
if token["type"] == "Characters":
text = token["data"]
new_tokens = []
end = 0
for match in self.url_re.finditer(text):
if match.start() > end:
new_tokens.append(
{"type": "Characters", "data": text[end : match.start()]}
)
url = match.group(0)
prefix = suffix = ""
# Sometimes we pick up too much in the url match, so look for
# bits we should drop and remove them from the match
url, prefix, suffix = self.strip_non_url_bits(url)
# If there's no protocol, add one
if PROTO_RE.search(url):
href = url
else:
href = "http://%s" % url
attrs = {(None, "href"): href, "_text": url}
attrs = self.apply_callbacks(attrs, True)
if attrs is None:
# Just add the text
new_tokens.append(
{"type": "Characters", "data": prefix + url + suffix}
)
else:
# Add the "a" tag!
if prefix:
new_tokens.append({"type": "Characters", "data": prefix})
_text = attrs.pop("_text", "")
new_tokens.extend(
[
{"type": "StartTag", "name": "a", "data": attrs},
{"type": "Characters", "data": str(_text)},
{"type": "EndTag", "name": "a"},
]
)
if suffix:
new_tokens.append({"type": "Characters", "data": suffix})
end = match.end()
if new_tokens:
# Yield the adjusted set of tokens and then continue
# through the loop
if end < len(text):
new_tokens.append({"type": "Characters", "data": text[end:]})
yield from new_tokens
continue
yield token
def handle_a_tag(self, token_buffer):
"""Handle the "a" tag
This could adjust the link or drop it altogether depending on what the
callbacks return.
This yields the new set of tokens.
"""
a_token = token_buffer[0]
if a_token["data"]:
attrs = a_token["data"]
else:
attrs = {}
text = self.extract_character_data(token_buffer)
attrs["_text"] = text
attrs = self.apply_callbacks(attrs, False)
if attrs is None:
# We're dropping the "a" tag and everything else and replacing
# it with character data. So emit that token.
yield {"type": "Characters", "data": text}
else:
new_text = attrs.pop("_text", "")
a_token["data"] = attrs
if text == new_text:
# The callbacks didn't change the text, so we yield the new "a"
# token, then whatever else was there, then the end "a" token
yield a_token
yield from token_buffer[1:]
else:
# If the callbacks changed the text, then we're going to drop
# all the tokens between the start and end "a" tags and replace
# it with the new text
yield a_token
yield {"type": "Characters", "data": str(new_text)}
yield token_buffer[-1]
def extract_entities(self, token):
"""Handles Characters tokens with entities
Our overridden tokenizer doesn't do anything with entities. However,
that means that the serializer will convert all ``&`` in Characters
tokens to ``&amp;``.
Since we don't want that, we extract entities here and convert them to
Entity tokens so the serializer will let them be.
:arg token: the Characters token to work on
:returns: generator of tokens
"""
data = token.get("data", "")
# If there isn't a & in the data, we can return now
if "&" not in data:
yield token
return
new_tokens = []
# For each possible entity that starts with a "&", we try to extract an
# actual entity and re-tokenize accordingly
for part in html5lib_shim.next_possible_entity(data):
if not part:
continue
if part.startswith("&"):
entity = html5lib_shim.match_entity(part)
if entity is not None:
if entity == "amp":
# LinkifyFilter can't match urls across token boundaries
# which is problematic with &amp; since that shows up in
# querystrings all the time. This special-cases &amp;
# and converts it to a & and sticks it in as a
# Characters token. It'll get merged with surrounding
# tokens in the BleachSanitizerfilter.__iter__ and
# escaped in the serializer.
new_tokens.append({"type": "Characters", "data": "&"})
else:
new_tokens.append({"type": "Entity", "name": entity})
# Length of the entity plus 2--one for & at the beginning
# and one for ; at the end
remainder = part[len(entity) + 2 :]
if remainder:
new_tokens.append({"type": "Characters", "data": remainder})
continue
new_tokens.append({"type": "Characters", "data": part})
yield from new_tokens
def __iter__(self):
in_a = False
in_skip_tag = None
token_buffer = []
for token in super().__iter__():
if in_a:
# Handle the case where we're in an "a" tag--we want to buffer tokens
# until we hit an end "a" tag.
if token["type"] == "EndTag" and token["name"] == "a":
# Add the end tag to the token buffer and then handle them
# and yield anything returned
token_buffer.append(token)
yield from self.handle_a_tag(token_buffer)
# Clear "a" related state and continue since we've yielded all
# the tokens we're going to yield
in_a = False
token_buffer = []
else:
token_buffer.extend(list(self.extract_entities(token)))
continue
if token["type"] in ["StartTag", "EmptyTag"]:
if token["name"] in self.skip_tags:
# Skip tags start a "special mode" where we don't linkify
# anything until the end tag.
in_skip_tag = token["name"]
elif token["name"] == "a":
# The "a" tag is special--we switch to a slurp mode and
# slurp all the tokens until the end "a" tag and then
# figure out what to do with them there.
in_a = True
token_buffer.append(token)
# We buffer the start tag, so we don't want to yield it,
# yet
continue
elif in_skip_tag and self.skip_tags:
# NOTE(willkg): We put this clause here since in_a and
# switching in and out of in_a takes precedence.
if token["type"] == "EndTag" and token["name"] == in_skip_tag:
in_skip_tag = None
elif not in_a and not in_skip_tag and token["type"] == "Characters":
new_stream = iter([token])
if self.parse_email:
new_stream = self.handle_email_addresses(new_stream)
new_stream = self.handle_links(new_stream)
for new_token in new_stream:
yield from self.extract_entities(new_token)
# We've already yielded this token, so continue
continue
yield token
@@ -0,0 +1 @@
from tensorboard._vendor.bleach._vendor.parse import urlparse # noqa
@@ -0,0 +1,638 @@
from itertools import chain
import re
import warnings
from xml.sax.saxutils import unescape
from tensorboard._vendor.bleach import html5lib_shim
from tensorboard._vendor.bleach import parse_shim
#: Set of allowed tags
ALLOWED_TAGS = frozenset(
(
"a",
"abbr",
"acronym",
"b",
"blockquote",
"code",
"em",
"i",
"li",
"ol",
"strong",
"ul",
)
)
#: Map of allowed attributes by tag
ALLOWED_ATTRIBUTES = {
"a": ["href", "title"],
"abbr": ["title"],
"acronym": ["title"],
}
#: List of allowed protocols
ALLOWED_PROTOCOLS = frozenset(("http", "https", "mailto"))
#: Invisible characters--0 to and including 31 except 9 (tab), 10 (lf), and 13 (cr)
INVISIBLE_CHARACTERS = "".join(
[chr(c) for c in chain(range(0, 9), range(11, 13), range(14, 32))]
)
#: Regexp for characters that are invisible
INVISIBLE_CHARACTERS_RE = re.compile("[" + INVISIBLE_CHARACTERS + "]", re.UNICODE)
#: String to replace invisible characters with. This can be a character, a
#: string, or even a function that takes a Python re matchobj
INVISIBLE_REPLACEMENT_CHAR = "?"
class NoCssSanitizerWarning(UserWarning):
pass
class Cleaner:
"""Cleaner for cleaning HTML fragments of malicious content
This cleaner is a security-focused function whose sole purpose is to remove
malicious content from a string such that it can be displayed as content in
a web page.
To use::
from bleach.sanitizer import Cleaner
cleaner = Cleaner()
for text in all_the_yucky_things:
sanitized = cleaner.clean(text)
.. Note::
This cleaner is not designed to use to transform content to be used in
non-web-page contexts.
.. Warning::
This cleaner is not thread-safe--the html parser has internal state.
Create a separate cleaner per thread!
"""
def __init__(
self,
tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
protocols=ALLOWED_PROTOCOLS,
strip=False,
strip_comments=True,
filters=None,
css_sanitizer=None,
):
"""Initializes a Cleaner
:arg set tags: set of allowed tags; defaults to
``bleach.sanitizer.ALLOWED_TAGS``
:arg dict attributes: allowed attributes; can be a callable, list or dict;
defaults to ``bleach.sanitizer.ALLOWED_ATTRIBUTES``
:arg list protocols: allowed list of protocols for links; defaults
to ``bleach.sanitizer.ALLOWED_PROTOCOLS``
:arg bool strip: whether or not to strip disallowed elements
:arg bool strip_comments: whether or not to strip HTML comments
:arg list filters: list of html5lib Filter classes to pass streamed content through
.. seealso:: http://html5lib.readthedocs.io/en/latest/movingparts.html#filters
.. Warning::
Using filters changes the output of ``bleach.Cleaner.clean``.
Make sure the way the filters change the output are secure.
:arg CSSSanitizer css_sanitizer: instance with a "sanitize_css" method for
sanitizing style attribute values and style text; defaults to None
"""
self.tags = tags
self.attributes = attributes
self.protocols = protocols
self.strip = strip
self.strip_comments = strip_comments
self.filters = filters or []
self.css_sanitizer = css_sanitizer
self.parser = html5lib_shim.BleachHTMLParser(
tags=self.tags,
strip=self.strip,
consume_entities=False,
namespaceHTMLElements=False,
)
self.walker = html5lib_shim.getTreeWalker("etree")
self.serializer = html5lib_shim.BleachHTMLSerializer(
quote_attr_values="always",
omit_optional_tags=False,
escape_lt_in_attrs=True,
# We want to leave entities as they are without escaping or
# resolving or expanding
resolve_entities=False,
# Bleach has its own sanitizer, so don't use the html5lib one
sanitize=False,
# clean preserves attr order
alphabetical_attributes=False,
)
if css_sanitizer is None:
# FIXME(willkg): this doesn't handle when attributes or an
# attributes value is a callable
attributes_values = []
if isinstance(attributes, list):
attributes_values = attributes
elif isinstance(attributes, dict):
attributes_values = []
for values in attributes.values():
if isinstance(values, (list, tuple)):
attributes_values.extend(values)
if "style" in attributes_values:
warnings.warn(
"'style' attribute specified, but css_sanitizer not set.",
category=NoCssSanitizerWarning,
)
def clean(self, text):
"""Cleans text and returns sanitized result as unicode
:arg str text: text to be cleaned
:returns: sanitized text as unicode
:raises TypeError: if ``text`` is not a text type
"""
if not isinstance(text, str):
message = (
f"argument cannot be of {text.__class__.__name__!r} type, "
+ "must be of text type"
)
raise TypeError(message)
if not text:
return ""
dom = self.parser.parseFragment(text)
filtered = BleachSanitizerFilter(
source=self.walker(dom),
allowed_tags=self.tags,
attributes=self.attributes,
strip_disallowed_tags=self.strip,
strip_html_comments=self.strip_comments,
css_sanitizer=self.css_sanitizer,
allowed_protocols=self.protocols,
)
# Apply any filters after the BleachSanitizerFilter
for filter_class in self.filters:
filtered = filter_class(source=filtered)
return self.serializer.render(filtered)
def attribute_filter_factory(attributes):
"""Generates attribute filter function for the given attributes value
The attributes value can take one of several shapes. This returns a filter
function appropriate to the attributes value. One nice thing about this is
that there's less if/then shenanigans in the ``allow_token`` method.
"""
if callable(attributes):
return attributes
if isinstance(attributes, dict):
def _attr_filter(tag, attr, value):
if tag in attributes:
attr_val = attributes[tag]
if callable(attr_val):
return attr_val(tag, attr, value)
if attr in attr_val:
return True
if "*" in attributes:
attr_val = attributes["*"]
if callable(attr_val):
return attr_val(tag, attr, value)
return attr in attr_val
return False
return _attr_filter
if isinstance(attributes, list):
def _attr_filter(tag, attr, value):
return attr in attributes
return _attr_filter
raise ValueError("attributes needs to be a callable, a list or a dict")
class BleachSanitizerFilter(html5lib_shim.SanitizerFilter):
"""html5lib Filter that sanitizes text
This filter can be used anywhere html5lib filters can be used.
"""
def __init__(
self,
source,
allowed_tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
allowed_protocols=ALLOWED_PROTOCOLS,
attr_val_is_uri=html5lib_shim.attr_val_is_uri,
svg_attr_val_allows_ref=html5lib_shim.svg_attr_val_allows_ref,
svg_allow_local_href=html5lib_shim.svg_allow_local_href,
strip_disallowed_tags=False,
strip_html_comments=True,
css_sanitizer=None,
):
"""Creates a BleachSanitizerFilter instance
:arg source: html5lib TreeWalker stream as an html5lib TreeWalker
:arg set allowed_tags: set of allowed tags; defaults to
``bleach.sanitizer.ALLOWED_TAGS``
:arg dict attributes: allowed attributes; can be a callable, list or dict;
defaults to ``bleach.sanitizer.ALLOWED_ATTRIBUTES``
:arg list allowed_protocols: allowed list of protocols for links; defaults
to ``bleach.sanitizer.ALLOWED_PROTOCOLS``
:arg attr_val_is_uri: set of attributes that have URI values
:arg svg_attr_val_allows_ref: set of SVG attributes that can have
references
:arg svg_allow_local_href: set of SVG elements that can have local
hrefs
:arg bool strip_disallowed_tags: whether or not to strip disallowed
tags
:arg bool strip_html_comments: whether or not to strip HTML comments
:arg CSSSanitizer css_sanitizer: instance with a "sanitize_css" method for
sanitizing style attribute values and style text; defaults to None
"""
# NOTE(willkg): This is the superclass of
# html5lib.filters.sanitizer.Filter. We call this directly skipping the
# __init__ for html5lib.filters.sanitizer.Filter because that does
# things we don't need to do and kicks up the deprecation warning for
# using Sanitizer.
html5lib_shim.Filter.__init__(self, source)
self.allowed_tags = frozenset(allowed_tags)
self.allowed_protocols = frozenset(allowed_protocols)
self.attr_filter = attribute_filter_factory(attributes)
self.strip_disallowed_tags = strip_disallowed_tags
self.strip_html_comments = strip_html_comments
self.attr_val_is_uri = attr_val_is_uri
self.svg_attr_val_allows_ref = svg_attr_val_allows_ref
self.css_sanitizer = css_sanitizer
self.svg_allow_local_href = svg_allow_local_href
def sanitize_stream(self, token_iterator):
for token in token_iterator:
ret = self.sanitize_token(token)
if not ret:
continue
if isinstance(ret, list):
yield from ret
else:
yield ret
def merge_characters(self, token_iterator):
"""Merge consecutive Characters tokens in a stream"""
characters_buffer = []
for token in token_iterator:
if characters_buffer:
if token["type"] == "Characters":
characters_buffer.append(token)
continue
else:
# Merge all the characters tokens together into one and then
# operate on it.
new_token = {
"data": "".join(
[char_token["data"] for char_token in characters_buffer]
),
"type": "Characters",
}
characters_buffer = []
yield new_token
elif token["type"] == "Characters":
characters_buffer.append(token)
continue
yield token
new_token = {
"data": "".join([char_token["data"] for char_token in characters_buffer]),
"type": "Characters",
}
yield new_token
def __iter__(self):
return self.merge_characters(
self.sanitize_stream(html5lib_shim.Filter.__iter__(self))
)
def sanitize_token(self, token):
"""Sanitize a token either by HTML-encoding or dropping.
Unlike sanitizer.Filter, allowed_attributes can be a dict of {'tag':
['attribute', 'pairs'], 'tag': callable}.
Here callable is a function with two arguments of attribute name and
value. It should return true of false.
Also gives the option to strip tags instead of encoding.
:arg dict token: token to sanitize
:returns: token or list of tokens
"""
token_type = token["type"]
if token_type in ["StartTag", "EndTag", "EmptyTag"]:
if token["name"] in self.allowed_tags:
return self.allow_token(token)
elif self.strip_disallowed_tags:
return None
else:
return self.disallowed_token(token)
elif token_type == "Comment":
if not self.strip_html_comments:
# call lxml.sax.saxutils to escape &, <, and > in addition to " and '
token["data"] = html5lib_shim.escape(
token["data"], entities={'"': "&quot;", "'": "&#x27;"}
)
return token
else:
return None
elif token_type == "Characters":
return self.sanitize_characters(token)
else:
return token
def sanitize_characters(self, token):
"""Handles Characters tokens
Our overridden tokenizer doesn't do anything with entities. However,
that means that the serializer will convert all ``&`` in Characters
tokens to ``&amp;``.
Since we don't want that, we extract entities here and convert them to
Entity tokens so the serializer will let them be.
:arg token: the Characters token to work on
:returns: a list of tokens
"""
data = token.get("data", "")
if not data:
return token
data = INVISIBLE_CHARACTERS_RE.sub(INVISIBLE_REPLACEMENT_CHAR, data)
token["data"] = data
# If there isn't a & in the data, we can return now
if "&" not in data:
return token
new_tokens = []
# For each possible entity that starts with a "&", we try to extract an
# actual entity and re-tokenize accordingly
for part in html5lib_shim.next_possible_entity(data):
if not part:
continue
if part.startswith("&"):
entity = html5lib_shim.match_entity(part)
if entity is not None:
if entity == "amp":
# LinkifyFilter can't match urls across token boundaries
# which is problematic with &amp; since that shows up in
# querystrings all the time. This special-cases &amp;
# and converts it to a & and sticks it in as a
# Characters token. It'll get merged with surrounding
# tokens in the BleachSanitizerfilter.__iter__ and
# escaped in the serializer.
new_tokens.append({"type": "Characters", "data": "&"})
else:
new_tokens.append({"type": "Entity", "name": entity})
# Length of the entity plus 2--one for & at the beginning
# and one for ; at the end
remainder = part[len(entity) + 2 :]
if remainder:
new_tokens.append({"type": "Characters", "data": remainder})
continue
new_tokens.append({"type": "Characters", "data": part})
return new_tokens
def sanitize_uri_value(self, value, allowed_protocols):
"""Checks a uri value to see if it's allowed
:arg value: the uri value to sanitize
:arg allowed_protocols: list of allowed protocols
:returns: allowed value or None
"""
# NOTE(willkg): This transforms the value into a normalized one that's
# easier to match and verify, but shouldn't get returned since it's
# vastly different than the original value.
# Convert all character entities in the value
normalized_uri = html5lib_shim.convert_entities(value)
# Nix backtick, space characters, and control characters
normalized_uri = re.sub(r"[`\000-\040\177-\240\s]+", "", normalized_uri)
# Remove REPLACEMENT characters
normalized_uri = normalized_uri.replace("\ufffd", "")
# Lowercase it--this breaks the value, but makes it easier to match
# against
normalized_uri = normalized_uri.lower()
try:
# Drop attributes with uri values that have protocols that aren't
# allowed
parsed = parse_shim.urlparse(normalized_uri)
except ValueError:
# URI is impossible to parse, therefore it's not allowed
return None
if parsed.scheme:
# If urlparse found a scheme, check that
if parsed.scheme in allowed_protocols:
return value
else:
# Allow uris that are just an anchor
if normalized_uri.startswith("#"):
return value
# Handle protocols that urlparse doesn't recognize like "myprotocol"
if (
":" in normalized_uri
and normalized_uri.split(":")[0] in allowed_protocols
):
return value
# If there's no protocol/scheme specified, then assume it's "http" or
# "https" and see if that's allowed
if "http" in allowed_protocols or "https" in allowed_protocols:
return value
return None
def allow_token(self, token):
"""Handles the case where we're allowing the tag"""
if "data" in token:
# Loop through all the attributes and drop the ones that are not
# allowed, are unsafe or break other rules. Additionally, fix
# attribute values that need fixing.
#
# At the end of this loop, we have the final set of attributes
# we're keeping.
attrs = {}
for namespaced_name, val in token["data"].items():
namespace, name = namespaced_name
# Drop attributes that are not explicitly allowed
#
# NOTE(willkg): We pass in the attribute name--not a namespaced
# name.
if not self.attr_filter(token["name"], name, val):
continue
# Drop attributes with uri values that use a disallowed protocol
# Sanitize attributes with uri values
if namespaced_name in self.attr_val_is_uri:
new_value = self.sanitize_uri_value(val, self.allowed_protocols)
if new_value is None:
continue
val = new_value
# Drop values in svg attrs with non-local IRIs
if namespaced_name in self.svg_attr_val_allows_ref:
new_val = re.sub(r"url\s*\(\s*[^#\s][^)]+?\)", " ", unescape(val))
new_val = new_val.strip()
if not new_val:
continue
else:
# Replace the val with the unescaped version because
# it's a iri
val = new_val
# Drop href and xlink:href attr for svg elements with non-local IRIs
if (None, token["name"]) in self.svg_allow_local_href:
if namespaced_name in [
(None, "href"),
(html5lib_shim.namespaces["xlink"], "href"),
]:
if re.search(r"^\s*[^#\s]", val):
continue
# If it's a style attribute, sanitize it
if namespaced_name == (None, "style"):
if self.css_sanitizer:
val = self.css_sanitizer.sanitize_css(val)
else:
# FIXME(willkg): if style is allowed, but no
# css_sanitizer was set up, then this is probably a
# mistake and we should raise an error here
#
# For now, we're going to set the value to "" because
# there was no sanitizer set
val = ""
# At this point, we want to keep the attribute, so add it in
attrs[namespaced_name] = val
token["data"] = attrs
return token
def disallowed_token(self, token):
token_type = token["type"]
if token_type == "EndTag":
token["data"] = f"</{token['name']}>"
elif token["data"]:
assert token_type in ("StartTag", "EmptyTag")
attrs = []
for (ns, name), v in token["data"].items():
# If we end up with a namespace, but no name, switch them so we
# have a valid name to use.
if ns and not name:
ns, name = name, ns
# Figure out namespaced name if the namespace is appropriate
# and exists; if the ns isn't in prefixes, then drop it.
if ns is None or ns not in html5lib_shim.prefixes:
namespaced_name = name
else:
namespaced_name = f"{html5lib_shim.prefixes[ns]}:{name}"
# NOTE(willkg): HTMLSerializer escapes attribute values
# already, so if we do it here (like HTMLSerializer does),
# then we end up double-escaping.
attrs.append(f' {namespaced_name}="{v}"')
token["data"] = f"<{token['name']}{''.join(attrs)}>"
else:
token["data"] = f"<{token['name']}>"
if token.get("selfClosing"):
token["data"] = f"{token['data'][:-1]}/>"
token["type"] = "Characters"
del token["name"]
return token
@@ -0,0 +1,19 @@
"""
Replacement module for what html5lib uses six for.
"""
import http.client
import operator
import urllib
PY3 = True
binary_type = bytes
string_types = (str,)
text_type = str
unichr = chr
viewkeys = operator.methodcaller("keys")
http_client = http.client
urllib = urllib
urllib_parse = urllib.parse
@@ -0,0 +1,342 @@
# coding: utf-8
"""
webencodings
~~~~~~~~~~~~
This is a Python implementation of the `WHATWG Encoding standard
<http://encoding.spec.whatwg.org/>`. See README for details.
:copyright: Copyright 2012 by Simon Sapin
:license: BSD, see LICENSE for details.
"""
from __future__ import unicode_literals
import codecs
from .labels import LABELS
VERSION = '0.5.1'
# Some names in Encoding are not valid Python aliases. Remap these.
PYTHON_NAMES = {
'iso-8859-8-i': 'iso-8859-8',
'x-mac-cyrillic': 'mac-cyrillic',
'macintosh': 'mac-roman',
'windows-874': 'cp874'}
CACHE = {}
def ascii_lower(string):
r"""Transform (only) ASCII letters to lower case: A-Z is mapped to a-z.
:param string: An Unicode string.
:returns: A new Unicode string.
This is used for `ASCII case-insensitive
<http://encoding.spec.whatwg.org/#ascii-case-insensitive>`_
matching of encoding labels.
The same matching is also used, among other things,
for `CSS keywords <http://dev.w3.org/csswg/css-values/#keywords>`_.
This is different from the :meth:`~py:str.lower` method of Unicode strings
which also affect non-ASCII characters,
sometimes mapping them into the ASCII range:
>>> keyword = u'Bac\N{KELVIN SIGN}ground'
>>> assert keyword.lower() == u'background'
>>> assert ascii_lower(keyword) != keyword.lower()
>>> assert ascii_lower(keyword) == u'bac\N{KELVIN SIGN}ground'
"""
# This turns out to be faster than unicode.translate()
return string.encode('utf8').lower().decode('utf8')
def lookup(label):
"""
Look for an encoding by its label.
This is the spec’s `get an encoding
<http://encoding.spec.whatwg.org/#concept-encoding-get>`_ algorithm.
Supported labels are listed there.
:param label: A string.
:returns:
An :class:`Encoding` object, or :obj:`None` for an unknown label.
"""
# Only strip ASCII whitespace: U+0009, U+000A, U+000C, U+000D, and U+0020.
label = ascii_lower(label.strip('\t\n\f\r '))
name = LABELS.get(label)
if name is None:
return None
encoding = CACHE.get(name)
if encoding is None:
if name == 'x-user-defined':
from .x_user_defined import codec_info
else:
python_name = PYTHON_NAMES.get(name, name)
# Any python_name value that gets to here should be valid.
codec_info = codecs.lookup(python_name)
encoding = Encoding(name, codec_info)
CACHE[name] = encoding
return encoding
def _get_encoding(encoding_or_label):
"""
Accept either an encoding object or label.
:param encoding: An :class:`Encoding` object or a label string.
:returns: An :class:`Encoding` object.
:raises: :exc:`~exceptions.LookupError` for an unknown label.
"""
if hasattr(encoding_or_label, 'codec_info'):
return encoding_or_label
encoding = lookup(encoding_or_label)
if encoding is None:
raise LookupError('Unknown encoding label: %r' % encoding_or_label)
return encoding
class Encoding(object):
"""Reresents a character encoding such as UTF-8,
that can be used for decoding or encoding.
.. attribute:: name
Canonical name of the encoding
.. attribute:: codec_info
The actual implementation of the encoding,
a stdlib :class:`~codecs.CodecInfo` object.
See :func:`codecs.register`.
"""
def __init__(self, name, codec_info):
self.name = name
self.codec_info = codec_info
def __repr__(self):
return '<Encoding %s>' % self.name
#: The UTF-8 encoding. Should be used for new content and formats.
UTF8 = lookup('utf-8')
_UTF16LE = lookup('utf-16le')
_UTF16BE = lookup('utf-16be')
def decode(input, fallback_encoding, errors='replace'):
"""
Decode a single string.
:param input: A byte string
:param fallback_encoding:
An :class:`Encoding` object or a label string.
The encoding to use if :obj:`input` does note have a BOM.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
:return:
A ``(output, encoding)`` tuple of an Unicode string
and an :obj:`Encoding`.
"""
# Fail early if `encoding` is an invalid label.
fallback_encoding = _get_encoding(fallback_encoding)
bom_encoding, input = _detect_bom(input)
encoding = bom_encoding or fallback_encoding
return encoding.codec_info.decode(input, errors)[0], encoding
def _detect_bom(input):
"""Return (bom_encoding, input), with any BOM removed from the input."""
if input.startswith(b'\xFF\xFE'):
return _UTF16LE, input[2:]
if input.startswith(b'\xFE\xFF'):
return _UTF16BE, input[2:]
if input.startswith(b'\xEF\xBB\xBF'):
return UTF8, input[3:]
return None, input
def encode(input, encoding=UTF8, errors='strict'):
"""
Encode a single string.
:param input: An Unicode string.
:param encoding: An :class:`Encoding` object or a label string.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
:return: A byte string.
"""
return _get_encoding(encoding).codec_info.encode(input, errors)[0]
def iter_decode(input, fallback_encoding, errors='replace'):
"""
"Pull"-based decoder.
:param input:
An iterable of byte strings.
The input is first consumed just enough to determine the encoding
based on the precense of a BOM,
then consumed on demand when the return value is.
:param fallback_encoding:
An :class:`Encoding` object or a label string.
The encoding to use if :obj:`input` does note have a BOM.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
:returns:
An ``(output, encoding)`` tuple.
:obj:`output` is an iterable of Unicode strings,
:obj:`encoding` is the :obj:`Encoding` that is being used.
"""
decoder = IncrementalDecoder(fallback_encoding, errors)
generator = _iter_decode_generator(input, decoder)
encoding = next(generator)
return generator, encoding
def _iter_decode_generator(input, decoder):
"""Return a generator that first yields the :obj:`Encoding`,
then yields output chukns as Unicode strings.
"""
decode = decoder.decode
input = iter(input)
for chunck in input:
output = decode(chunck)
if output:
assert decoder.encoding is not None
yield decoder.encoding
yield output
break
else:
# Input exhausted without determining the encoding
output = decode(b'', final=True)
assert decoder.encoding is not None
yield decoder.encoding
if output:
yield output
return
for chunck in input:
output = decode(chunck)
if output:
yield output
output = decode(b'', final=True)
if output:
yield output
def iter_encode(input, encoding=UTF8, errors='strict'):
"""
“Pull”-based encoder.
:param input: An iterable of Unicode strings.
:param encoding: An :class:`Encoding` object or a label string.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
:returns: An iterable of byte strings.
"""
# Fail early if `encoding` is an invalid label.
encode = IncrementalEncoder(encoding, errors).encode
return _iter_encode_generator(input, encode)
def _iter_encode_generator(input, encode):
for chunck in input:
output = encode(chunck)
if output:
yield output
output = encode('', final=True)
if output:
yield output
class IncrementalDecoder(object):
"""
“Push”-based decoder.
:param fallback_encoding:
An :class:`Encoding` object or a label string.
The encoding to use if :obj:`input` does note have a BOM.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
"""
def __init__(self, fallback_encoding, errors='replace'):
# Fail early if `encoding` is an invalid label.
self._fallback_encoding = _get_encoding(fallback_encoding)
self._errors = errors
self._buffer = b''
self._decoder = None
#: The actual :class:`Encoding` that is being used,
#: or :obj:`None` if that is not determined yet.
#: (Ie. if there is not enough input yet to determine
#: if there is a BOM.)
self.encoding = None # Not known yet.
def decode(self, input, final=False):
"""Decode one chunk of the input.
:param input: A byte string.
:param final:
Indicate that no more input is available.
Must be :obj:`True` if this is the last call.
:returns: An Unicode string.
"""
decoder = self._decoder
if decoder is not None:
return decoder(input, final)
input = self._buffer + input
encoding, input = _detect_bom(input)
if encoding is None:
if len(input) < 3 and not final: # Not enough data yet.
self._buffer = input
return ''
else: # No BOM
encoding = self._fallback_encoding
decoder = encoding.codec_info.incrementaldecoder(self._errors).decode
self._decoder = decoder
self.encoding = encoding
return decoder(input, final)
class IncrementalEncoder(object):
"""
“Push”-based encoder.
:param encoding: An :class:`Encoding` object or a label string.
:param errors: Type of error handling. See :func:`codecs.register`.
:raises: :exc:`~exceptions.LookupError` for an unknown encoding label.
.. method:: encode(input, final=False)
:param input: An Unicode string.
:param final:
Indicate that no more input is available.
Must be :obj:`True` if this is the last call.
:returns: A byte string.
"""
def __init__(self, encoding=UTF8, errors='strict'):
encoding = _get_encoding(encoding)
self.encode = encoding.codec_info.incrementalencoder(errors).encode
@@ -0,0 +1,231 @@
"""
webencodings.labels
~~~~~~~~~~~~~~~~~~~
Map encoding labels to their name.
:copyright: Copyright 2012 by Simon Sapin
:license: BSD, see LICENSE for details.
"""
# XXX Do not edit!
# This file is automatically generated by mklabels.py
LABELS = {
'unicode-1-1-utf-8': 'utf-8',
'utf-8': 'utf-8',
'utf8': 'utf-8',
'866': 'ibm866',
'cp866': 'ibm866',
'csibm866': 'ibm866',
'ibm866': 'ibm866',
'csisolatin2': 'iso-8859-2',
'iso-8859-2': 'iso-8859-2',
'iso-ir-101': 'iso-8859-2',
'iso8859-2': 'iso-8859-2',
'iso88592': 'iso-8859-2',
'iso_8859-2': 'iso-8859-2',
'iso_8859-2:1987': 'iso-8859-2',
'l2': 'iso-8859-2',
'latin2': 'iso-8859-2',
'csisolatin3': 'iso-8859-3',
'iso-8859-3': 'iso-8859-3',
'iso-ir-109': 'iso-8859-3',
'iso8859-3': 'iso-8859-3',
'iso88593': 'iso-8859-3',
'iso_8859-3': 'iso-8859-3',
'iso_8859-3:1988': 'iso-8859-3',
'l3': 'iso-8859-3',
'latin3': 'iso-8859-3',
'csisolatin4': 'iso-8859-4',
'iso-8859-4': 'iso-8859-4',
'iso-ir-110': 'iso-8859-4',
'iso8859-4': 'iso-8859-4',
'iso88594': 'iso-8859-4',
'iso_8859-4': 'iso-8859-4',
'iso_8859-4:1988': 'iso-8859-4',
'l4': 'iso-8859-4',
'latin4': 'iso-8859-4',
'csisolatincyrillic': 'iso-8859-5',
'cyrillic': 'iso-8859-5',
'iso-8859-5': 'iso-8859-5',
'iso-ir-144': 'iso-8859-5',
'iso8859-5': 'iso-8859-5',
'iso88595': 'iso-8859-5',
'iso_8859-5': 'iso-8859-5',
'iso_8859-5:1988': 'iso-8859-5',
'arabic': 'iso-8859-6',
'asmo-708': 'iso-8859-6',
'csiso88596e': 'iso-8859-6',
'csiso88596i': 'iso-8859-6',
'csisolatinarabic': 'iso-8859-6',
'ecma-114': 'iso-8859-6',
'iso-8859-6': 'iso-8859-6',
'iso-8859-6-e': 'iso-8859-6',
'iso-8859-6-i': 'iso-8859-6',
'iso-ir-127': 'iso-8859-6',
'iso8859-6': 'iso-8859-6',
'iso88596': 'iso-8859-6',
'iso_8859-6': 'iso-8859-6',
'iso_8859-6:1987': 'iso-8859-6',
'csisolatingreek': 'iso-8859-7',
'ecma-118': 'iso-8859-7',
'elot_928': 'iso-8859-7',
'greek': 'iso-8859-7',
'greek8': 'iso-8859-7',
'iso-8859-7': 'iso-8859-7',
'iso-ir-126': 'iso-8859-7',
'iso8859-7': 'iso-8859-7',
'iso88597': 'iso-8859-7',
'iso_8859-7': 'iso-8859-7',
'iso_8859-7:1987': 'iso-8859-7',
'sun_eu_greek': 'iso-8859-7',
'csiso88598e': 'iso-8859-8',
'csisolatinhebrew': 'iso-8859-8',
'hebrew': 'iso-8859-8',
'iso-8859-8': 'iso-8859-8',
'iso-8859-8-e': 'iso-8859-8',
'iso-ir-138': 'iso-8859-8',
'iso8859-8': 'iso-8859-8',
'iso88598': 'iso-8859-8',
'iso_8859-8': 'iso-8859-8',
'iso_8859-8:1988': 'iso-8859-8',
'visual': 'iso-8859-8',
'csiso88598i': 'iso-8859-8-i',
'iso-8859-8-i': 'iso-8859-8-i',
'logical': 'iso-8859-8-i',
'csisolatin6': 'iso-8859-10',
'iso-8859-10': 'iso-8859-10',
'iso-ir-157': 'iso-8859-10',
'iso8859-10': 'iso-8859-10',
'iso885910': 'iso-8859-10',
'l6': 'iso-8859-10',
'latin6': 'iso-8859-10',
'iso-8859-13': 'iso-8859-13',
'iso8859-13': 'iso-8859-13',
'iso885913': 'iso-8859-13',
'iso-8859-14': 'iso-8859-14',
'iso8859-14': 'iso-8859-14',
'iso885914': 'iso-8859-14',
'csisolatin9': 'iso-8859-15',
'iso-8859-15': 'iso-8859-15',
'iso8859-15': 'iso-8859-15',
'iso885915': 'iso-8859-15',
'iso_8859-15': 'iso-8859-15',
'l9': 'iso-8859-15',
'iso-8859-16': 'iso-8859-16',
'cskoi8r': 'koi8-r',
'koi': 'koi8-r',
'koi8': 'koi8-r',
'koi8-r': 'koi8-r',
'koi8_r': 'koi8-r',
'koi8-u': 'koi8-u',
'csmacintosh': 'macintosh',
'mac': 'macintosh',
'macintosh': 'macintosh',
'x-mac-roman': 'macintosh',
'dos-874': 'windows-874',
'iso-8859-11': 'windows-874',
'iso8859-11': 'windows-874',
'iso885911': 'windows-874',
'tis-620': 'windows-874',
'windows-874': 'windows-874',
'cp1250': 'windows-1250',
'windows-1250': 'windows-1250',
'x-cp1250': 'windows-1250',
'cp1251': 'windows-1251',
'windows-1251': 'windows-1251',
'x-cp1251': 'windows-1251',
'ansi_x3.4-1968': 'windows-1252',
'ascii': 'windows-1252',
'cp1252': 'windows-1252',
'cp819': 'windows-1252',
'csisolatin1': 'windows-1252',
'ibm819': 'windows-1252',
'iso-8859-1': 'windows-1252',
'iso-ir-100': 'windows-1252',
'iso8859-1': 'windows-1252',
'iso88591': 'windows-1252',
'iso_8859-1': 'windows-1252',
'iso_8859-1:1987': 'windows-1252',
'l1': 'windows-1252',
'latin1': 'windows-1252',
'us-ascii': 'windows-1252',
'windows-1252': 'windows-1252',
'x-cp1252': 'windows-1252',
'cp1253': 'windows-1253',
'windows-1253': 'windows-1253',
'x-cp1253': 'windows-1253',
'cp1254': 'windows-1254',
'csisolatin5': 'windows-1254',
'iso-8859-9': 'windows-1254',
'iso-ir-148': 'windows-1254',
'iso8859-9': 'windows-1254',
'iso88599': 'windows-1254',
'iso_8859-9': 'windows-1254',
'iso_8859-9:1989': 'windows-1254',
'l5': 'windows-1254',
'latin5': 'windows-1254',
'windows-1254': 'windows-1254',
'x-cp1254': 'windows-1254',
'cp1255': 'windows-1255',
'windows-1255': 'windows-1255',
'x-cp1255': 'windows-1255',
'cp1256': 'windows-1256',
'windows-1256': 'windows-1256',
'x-cp1256': 'windows-1256',
'cp1257': 'windows-1257',
'windows-1257': 'windows-1257',
'x-cp1257': 'windows-1257',
'cp1258': 'windows-1258',
'windows-1258': 'windows-1258',
'x-cp1258': 'windows-1258',
'x-mac-cyrillic': 'x-mac-cyrillic',
'x-mac-ukrainian': 'x-mac-cyrillic',
'chinese': 'gbk',
'csgb2312': 'gbk',
'csiso58gb231280': 'gbk',
'gb2312': 'gbk',
'gb_2312': 'gbk',
'gb_2312-80': 'gbk',
'gbk': 'gbk',
'iso-ir-58': 'gbk',
'x-gbk': 'gbk',
'gb18030': 'gb18030',
'hz-gb-2312': 'hz-gb-2312',
'big5': 'big5',
'big5-hkscs': 'big5',
'cn-big5': 'big5',
'csbig5': 'big5',
'x-x-big5': 'big5',
'cseucpkdfmtjapanese': 'euc-jp',
'euc-jp': 'euc-jp',
'x-euc-jp': 'euc-jp',
'csiso2022jp': 'iso-2022-jp',
'iso-2022-jp': 'iso-2022-jp',
'csshiftjis': 'shift_jis',
'ms_kanji': 'shift_jis',
'shift-jis': 'shift_jis',
'shift_jis': 'shift_jis',
'sjis': 'shift_jis',
'windows-31j': 'shift_jis',
'x-sjis': 'shift_jis',
'cseuckr': 'euc-kr',
'csksc56011987': 'euc-kr',
'euc-kr': 'euc-kr',
'iso-ir-149': 'euc-kr',
'korean': 'euc-kr',
'ks_c_5601-1987': 'euc-kr',
'ks_c_5601-1989': 'euc-kr',
'ksc5601': 'euc-kr',
'ksc_5601': 'euc-kr',
'windows-949': 'euc-kr',
'csiso2022kr': 'iso-2022-kr',
'iso-2022-kr': 'iso-2022-kr',
'utf-16be': 'utf-16be',
'utf-16': 'utf-16le',
'utf-16le': 'utf-16le',
'x-user-defined': 'x-user-defined',
}
@@ -0,0 +1,59 @@
"""
webencodings.mklabels
~~~~~~~~~~~~~~~~~~~~~
Regenarate the webencodings.labels module.
:copyright: Copyright 2012 by Simon Sapin
:license: BSD, see LICENSE for details.
"""
import json
try:
from urllib import urlopen
except ImportError:
from urllib.request import urlopen
def assert_lower(string):
assert string == string.lower()
return string
def generate(url):
parts = ['''\
"""
webencodings.labels
~~~~~~~~~~~~~~~~~~~
Map encoding labels to their name.
:copyright: Copyright 2012 by Simon Sapin
:license: BSD, see LICENSE for details.
"""
# XXX Do not edit!
# This file is automatically generated by mklabels.py
LABELS = {
''']
labels = [
(repr(assert_lower(label)).lstrip('u'),
repr(encoding['name']).lstrip('u'))
for category in json.loads(urlopen(url).read().decode('ascii'))
for encoding in category['encodings']
for label in encoding['labels']]
max_len = max(len(label) for label, name in labels)
parts.extend(
' %s:%s %s,\n' % (label, ' ' * (max_len - len(label)), name)
for label, name in labels)
parts.append('}')
return ''.join(parts)
if __name__ == '__main__':
print(generate('http://encoding.spec.whatwg.org/encodings.json'))
@@ -0,0 +1,325 @@
# coding: utf-8
"""
webencodings.x_user_defined
~~~~~~~~~~~~~~~~~~~~~~~~~~~
An implementation of the x-user-defined encoding.
:copyright: Copyright 2012 by Simon Sapin
:license: BSD, see LICENSE for details.
"""
from __future__ import unicode_literals
import codecs
### Codec APIs
class Codec(codecs.Codec):
def encode(self, input, errors='strict'):
return codecs.charmap_encode(input, errors, encoding_table)
def decode(self, input, errors='strict'):
return codecs.charmap_decode(input, errors, decoding_table)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return codecs.charmap_encode(input, self.errors, encoding_table)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return codecs.charmap_decode(input, self.errors, decoding_table)[0]
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
### encodings module API
codec_info = codecs.CodecInfo(
name='x-user-defined',
encode=Codec().encode,
decode=Codec().decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)
### Decoding Table
# Python 3:
# for c in range(256): print(' %r' % chr(c if c < 128 else c + 0xF700))
decoding_table = (
'\x00'
'\x01'
'\x02'
'\x03'
'\x04'
'\x05'
'\x06'
'\x07'
'\x08'
'\t'
'\n'
'\x0b'
'\x0c'
'\r'
'\x0e'
'\x0f'
'\x10'
'\x11'
'\x12'
'\x13'
'\x14'
'\x15'
'\x16'
'\x17'
'\x18'
'\x19'
'\x1a'
'\x1b'
'\x1c'
'\x1d'
'\x1e'
'\x1f'
' '
'!'
'"'
'#'
'$'
'%'
'&'
"'"
'('
')'
'*'
'+'
','
'-'
'.'
'/'
'0'
'1'
'2'
'3'
'4'
'5'
'6'
'7'
'8'
'9'
':'
';'
'<'
'='
'>'
'?'
'@'
'A'
'B'
'C'
'D'
'E'
'F'
'G'
'H'
'I'
'J'
'K'
'L'
'M'
'N'
'O'
'P'
'Q'
'R'
'S'
'T'
'U'
'V'
'W'
'X'
'Y'
'Z'
'['
'\\'
']'
'^'
'_'
'`'
'a'
'b'
'c'
'd'
'e'
'f'
'g'
'h'
'i'
'j'
'k'
'l'
'm'
'n'
'o'
'p'
'q'
'r'
's'
't'
'u'
'v'
'w'
'x'
'y'
'z'
'{'
'|'
'}'
'~'
'\x7f'
'\uf780'
'\uf781'
'\uf782'
'\uf783'
'\uf784'
'\uf785'
'\uf786'
'\uf787'
'\uf788'
'\uf789'
'\uf78a'
'\uf78b'
'\uf78c'
'\uf78d'
'\uf78e'
'\uf78f'
'\uf790'
'\uf791'
'\uf792'
'\uf793'
'\uf794'
'\uf795'
'\uf796'
'\uf797'
'\uf798'
'\uf799'
'\uf79a'
'\uf79b'
'\uf79c'
'\uf79d'
'\uf79e'
'\uf79f'
'\uf7a0'
'\uf7a1'
'\uf7a2'
'\uf7a3'
'\uf7a4'
'\uf7a5'
'\uf7a6'
'\uf7a7'
'\uf7a8'
'\uf7a9'
'\uf7aa'
'\uf7ab'
'\uf7ac'
'\uf7ad'
'\uf7ae'
'\uf7af'
'\uf7b0'
'\uf7b1'
'\uf7b2'
'\uf7b3'
'\uf7b4'
'\uf7b5'
'\uf7b6'
'\uf7b7'
'\uf7b8'
'\uf7b9'
'\uf7ba'
'\uf7bb'
'\uf7bc'
'\uf7bd'
'\uf7be'
'\uf7bf'
'\uf7c0'
'\uf7c1'
'\uf7c2'
'\uf7c3'
'\uf7c4'
'\uf7c5'
'\uf7c6'
'\uf7c7'
'\uf7c8'
'\uf7c9'
'\uf7ca'
'\uf7cb'
'\uf7cc'
'\uf7cd'
'\uf7ce'
'\uf7cf'
'\uf7d0'
'\uf7d1'
'\uf7d2'
'\uf7d3'
'\uf7d4'
'\uf7d5'
'\uf7d6'
'\uf7d7'
'\uf7d8'
'\uf7d9'
'\uf7da'
'\uf7db'
'\uf7dc'
'\uf7dd'
'\uf7de'
'\uf7df'
'\uf7e0'
'\uf7e1'
'\uf7e2'
'\uf7e3'
'\uf7e4'
'\uf7e5'
'\uf7e6'
'\uf7e7'
'\uf7e8'
'\uf7e9'
'\uf7ea'
'\uf7eb'
'\uf7ec'
'\uf7ed'
'\uf7ee'
'\uf7ef'
'\uf7f0'
'\uf7f1'
'\uf7f2'
'\uf7f3'
'\uf7f4'
'\uf7f5'
'\uf7f6'
'\uf7f7'
'\uf7f8'
'\uf7f9'
'\uf7fa'
'\uf7fb'
'\uf7fc'
'\uf7fd'
'\uf7fe'
'\uf7ff'
)
### Encoding table
encoding_table = codecs.charmap_build(decoding_table)
@@ -0,0 +1,36 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Bindings for TensorBoard frontend assets."""
import os
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
def get_default_assets_zip_provider():
"""Try to get a function to provide frontend assets.
Returns:
Either (a) a callable that takes no arguments and returns an open
file handle to a Zip archive of frontend assets, or (b) `None`, if
the frontend assets cannot be found.
"""
path = os.path.join(os.path.dirname(__file__), "webfiles.zip")
if not os.path.exists(path):
logger.warning("webfiles.zip static assets not found: %s", path)
return None
return lambda: open(path, "rb")
@@ -0,0 +1,102 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Experimental framework for authentication in TensorBoard."""
import abc
class AuthProvider(metaclass=abc.ABCMeta):
"""Authentication provider for a specific kind of credential."""
def authenticate(self, environ):
"""Produce an opaque auth token from a WSGI request environment.
Args:
environ: A WSGI environment `dict`; see PEP 3333.
Returns:
A Python object representing an auth token. The representation
and semantics depend on the particular `AuthProvider`
implementation.
Raises:
Exception: Any error, usually `tensorboard.errors.PublicError`
subclasses (like `PermissionDenied`) but also possibly a
custom error type that should propagate to a WSGI middleware
for effecting a redirect-driven auth flow.
"""
pass
class AuthContext:
"""Authentication context within the scope of a single request.
Auth providers are keyed within an `AuthContext` by arbitrary
unique keys. It may often make sense for the key used for an
auth provider to simply be that provider's type object.
"""
def __init__(self, providers, environ):
"""Create an auth context.
Args:
providers: A mapping from provider keys (opaque values) to
`AuthProvider` implementations.
environ: A WSGI environment (see PEP 3333).
"""
self._environ = environ
self._providers = providers
self._cache = {}
@classmethod
def empty(cls):
"""Create an auth context with no registered providers.
Returns:
A new `AuthContext` value for which any call to `get` will
fail with a `KeyError`.
"""
# Use an empty dict for the environ. This is not a valid WSGI
# environment, but it doesn't matter because it's never used.
return cls({}, {})
def get(self, provider_key):
"""Get an auth token from the auth provider with the given key.
If successful, the result will be cached on this auth context.
If unsuccessful, nothing will be cached, so a future call will
invoke the underlying `AuthProvider.authenticate` method again.
This method is not thread-safe. If multiple threads share an
auth context for a single request, then they must synchronize
externally when calling this method.
Returns:
The result of `provider.authenticate(...)` for the auth
provider specified by `provider_key`.
Raises:
KeyError: If the given `provider_key` does not correspond to
any registered `AuthProvider`.
Exception: As raised by the underlying `AuthProvider`.
"""
provider = self._providers[provider_key]
sentinel = object()
value = self._cache.get(provider_key, sentinel)
if value is not sentinel:
return value
value = provider.authenticate(self._environ)
self._cache[provider_key] = value
return value
@@ -0,0 +1,605 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorBoard WSGI Application Logic.
Provides TensorBoardWSGIApp for building a TensorBoard WSGI app.
"""
import base64
import collections
import hashlib
import io
import json
import re
import textwrap
import time
from urllib import parse as urlparse
import zipfile
from werkzeug import wrappers
from tensorboard import errors
from tensorboard import plugin_util
from tensorboard.backend import auth_context_middleware
from tensorboard.backend import client_feature_flags
from tensorboard.backend import empty_path_redirect
from tensorboard.backend import experiment_id
from tensorboard.backend import experimental_plugin
from tensorboard.backend import http_util
from tensorboard.backend import path_prefix
from tensorboard.backend import security_validator
from tensorboard.plugins import base_plugin
from tensorboard.plugins.core import core_plugin
from tensorboard.util import tb_logging
DATA_PREFIX = "/data"
PLUGIN_PREFIX = "/plugin"
PLUGINS_LISTING_ROUTE = "/plugins_listing"
PLUGIN_ENTRY_ROUTE = "/plugin_entry.html"
EXPERIMENTAL_PLUGINS_QUERY_PARAM = "experimentalPlugin"
# Slashes in a plugin name could throw the router for a loop. An empty
# name would be confusing, too. To be safe, let's restrict the valid
# names as follows.
_VALID_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_-]+$")
logger = tb_logging.get_logger()
def TensorBoardWSGIApp(
flags,
plugins,
data_provider=None,
assets_zip_provider=None,
deprecated_multiplexer=None,
auth_providers=None,
experimental_middlewares=None,
):
"""Constructs a TensorBoard WSGI app from plugins and data providers.
Args:
flags: An argparse.Namespace containing TensorBoard CLI flags.
plugins: A list of plugins, which can be provided as TBPlugin subclasses
or TBLoader instances or subclasses.
data_provider: Instance of `tensorboard.data.provider.DataProvider`. May
be `None` if `flags.generic_data` is set to `"false"` in which case
`deprecated_multiplexer` must be passed instead.
assets_zip_provider: See TBContext documentation for more information. If
`None` a placeholder assets zipfile will be used containing only a
default `index.html` file, and the actual frontend assets must be
supplied by middleware wrapping this WSGI app.
deprecated_multiplexer: Optional `plugin_event_multiplexer.EventMultiplexer`
to use for any plugins not yet enabled for the DataProvider API.
Required if the data_provider argument is not passed.
auth_providers: Optional mapping whose values are `AuthProvider` values
and whose keys are used by (e.g.) data providers to specify
`AuthProvider`s via the `AuthContext.get` interface. Defaults to `{}`.
experimental_middlewares: Optional list of WSGI middlewares (i.e.,
callables that take a WSGI application and return a WSGI application)
to apply directly around the core TensorBoard app itself, "inside" the
request redirection machinery for `--path_prefix`, experiment IDs, etc.
You can use this to add handlers for additional routes. Middlewares are
applied in listed order, so the first element of this list is the
innermost application. Defaults to `[]`. This parameter is experimental
and may be reworked or removed.
Returns:
A WSGI application that implements the TensorBoard backend.
:type plugins: list[base_plugin.TBLoader]
"""
if assets_zip_provider is None:
assets_zip_provider = _placeholder_assets_zip_provider
plugin_name_to_instance = {}
context = base_plugin.TBContext(
data_provider=data_provider,
flags=flags,
logdir=flags.logdir,
multiplexer=deprecated_multiplexer,
assets_zip_provider=assets_zip_provider,
plugin_name_to_instance=plugin_name_to_instance,
sampling_hints=flags.samples_per_plugin,
window_title=flags.window_title,
)
tbplugins = []
experimental_plugins = []
for plugin_spec in plugins:
loader = make_plugin_loader(plugin_spec)
try:
plugin = loader.load(context)
except Exception:
logger.error(
"Failed to load plugin %s; ignoring it.",
getattr(loader.load, "__qualname__", loader.load),
exc_info=True,
)
plugin = None
if plugin is None:
continue
tbplugins.append(plugin)
if isinstance(
loader, experimental_plugin.ExperimentalPlugin
) or isinstance(plugin, experimental_plugin.ExperimentalPlugin):
experimental_plugins.append(plugin.plugin_name)
plugin_name_to_instance[plugin.plugin_name] = plugin
return TensorBoardWSGI(
tbplugins,
flags.path_prefix,
data_provider,
experimental_plugins,
auth_providers,
experimental_middlewares,
)
def make_plugin_loader(plugin_spec):
"""Returns a plugin loader for the given plugin.
Args:
plugin_spec: A TBPlugin subclass, or a TBLoader instance or subclass.
Returns:
A TBLoader for the given plugin.
:type plugin_spec:
Type[base_plugin.TBPlugin] | Type[base_plugin.TBLoader] |
base_plugin.TBLoader
:rtype: base_plugin.TBLoader
"""
if isinstance(plugin_spec, base_plugin.TBLoader):
return plugin_spec
if isinstance(plugin_spec, type):
if issubclass(plugin_spec, base_plugin.TBLoader):
return plugin_spec()
if issubclass(plugin_spec, base_plugin.TBPlugin):
return base_plugin.BasicLoader(plugin_spec)
raise TypeError("Not a TBLoader or TBPlugin subclass: %r" % (plugin_spec,))
class TensorBoardWSGI:
"""The TensorBoard WSGI app that delegates to a set of TBPlugin."""
def __init__(
self,
plugins,
path_prefix="",
data_provider=None,
experimental_plugins=None,
auth_providers=None,
experimental_middlewares=None,
):
"""Constructs TensorBoardWSGI instance.
Args:
plugins: A list of base_plugin.TBPlugin subclass instances.
path_prefix: A prefix of the path when app isn't served from root.
data_provider: `tensorboard.data.provider.DataProvider` or
`None`; if present, will inform the "active" state of
`/plugins_listing`.
experimental_plugins: A list of plugin names that are only provided
experimentally. The corresponding plugins will only be activated for
a user if the user has specified the plugin with the experimentalPlugin
query parameter in the URL.
auth_providers: Optional mapping whose values are `AuthProvider`
values and whose keys are used by (e.g.) data providers to specify
`AuthProvider`s via the `AuthContext.get` interface.
Defaults to `{}`.
experimental_middlewares: Optional list of WSGI middlewares to apply
directly around the core TensorBoard app itself. Defaults to `[]`.
This parameter is experimental and may be reworked or removed.
Returns:
A WSGI application for the set of all TBPlugin instances.
Raises:
ValueError: If some plugin has no plugin_name
ValueError: If some plugin has an invalid plugin_name (plugin
names must only contain [A-Za-z0-9_.-])
ValueError: If two plugins have the same plugin_name
ValueError: If some plugin handles a route that does not start
with a slash
:type plugins: list[base_plugin.TBPlugin]
"""
self._plugins = plugins
self._path_prefix = path_prefix
self._data_provider = data_provider
self._experimental_plugins = frozenset(experimental_plugins or ())
self._auth_providers = auth_providers or {}
self._extra_middlewares = list(experimental_middlewares or [])
if self._path_prefix.endswith("/"):
# Should have been fixed by `fix_flags`.
raise ValueError(
"Trailing slash in path prefix: %r" % self._path_prefix
)
self.exact_routes = {
# TODO(@chihuahua): Delete this RPC once we have skylark rules that
# obviate the need for the frontend to determine which plugins are
# active.
DATA_PREFIX + PLUGINS_LISTING_ROUTE: self._serve_plugins_listing,
DATA_PREFIX + PLUGIN_ENTRY_ROUTE: self._serve_plugin_entry,
}
unordered_prefix_routes = {}
# Serve the routes from the registered plugins using their name as the route
# prefix. For example if plugin z has two routes /a and /b, they will be
# served as /data/plugin/z/a and /data/plugin/z/b.
plugin_names_encountered = set()
for plugin in self._plugins:
if plugin.plugin_name is None:
raise ValueError("Plugin %s has no plugin_name" % plugin)
if not _VALID_PLUGIN_RE.match(plugin.plugin_name):
raise ValueError(
"Plugin %s has invalid name %r"
% (plugin, plugin.plugin_name)
)
if plugin.plugin_name in plugin_names_encountered:
raise ValueError(
"Duplicate plugins for name %s" % plugin.plugin_name
)
plugin_names_encountered.add(plugin.plugin_name)
try:
plugin_apps = plugin.get_plugin_apps()
except Exception as e: # pylint: disable=broad-except
if (
type(plugin) is core_plugin.CorePlugin
): # pylint: disable=unidiomatic-typecheck
raise
logger.warning(
"Plugin %s failed. Exception: %s",
plugin.plugin_name,
str(e),
)
continue
for route, app in plugin_apps.items():
if not route.startswith("/"):
raise ValueError(
"Plugin named %r handles invalid route %r: "
"route does not start with a slash"
% (plugin.plugin_name, route)
)
if (
type(plugin) is core_plugin.CorePlugin
): # pylint: disable=unidiomatic-typecheck
path = route
else:
path = (
DATA_PREFIX
+ PLUGIN_PREFIX
+ "/"
+ plugin.plugin_name
+ route
)
if path.endswith("/*"):
# Note we remove the '*' but leave the slash in place.
path = path[:-1]
if "*" in path:
# note we re-add the removed * in the format string
raise ValueError(
"Plugin %r handles invalid route '%s*': Only "
"trailing wildcards are supported "
"(i.e., `/.../*`)" % (plugin.plugin_name, path)
)
unordered_prefix_routes[path] = app
else:
if "*" in path:
raise ValueError(
"Plugin %r handles invalid route %r: Only "
"trailing wildcards are supported "
"(i.e., `/.../*`)" % (plugin.plugin_name, path)
)
self.exact_routes[path] = app
# Wildcard routes will be checked in the given order, so we sort them
# longest to shortest so that a more specific route will take precedence
# over a more general one (e.g., a catchall route `/*` should come last).
self.prefix_routes = collections.OrderedDict(
sorted(
unordered_prefix_routes.items(),
key=lambda x: len(x[0]),
reverse=True,
)
)
self._app = self._create_wsgi_app()
def _create_wsgi_app(self):
"""Apply middleware to create the final WSGI app."""
app = self._route_request
for middleware in self._extra_middlewares:
app = middleware(app)
app = auth_context_middleware.AuthContextMiddleware(
app, self._auth_providers
)
app = client_feature_flags.ClientFeatureFlagsMiddleware(app)
app = empty_path_redirect.EmptyPathRedirectMiddleware(app)
app = experiment_id.ExperimentIdMiddleware(app)
app = path_prefix.PathPrefixMiddleware(app, self._path_prefix)
app = security_validator.SecurityValidatorMiddleware(app)
app = _handling_errors(app)
return app
@wrappers.Request.application
def _serve_plugin_entry(self, request):
"""Serves a HTML for iframed plugin entry point.
Args:
request: The werkzeug.Request object.
Returns:
A werkzeug.Response object.
"""
name = request.args.get("name")
plugins = [
plugin for plugin in self._plugins if plugin.plugin_name == name
]
if not plugins:
raise errors.NotFoundError(name)
if len(plugins) > 1:
# Technically is not possible as plugin names are unique and is checked
# by the check on __init__.
reason = (
"Plugin invariant error: multiple plugins with name "
"{name} found: {list}"
).format(name=name, list=plugins)
raise AssertionError(reason)
plugin = plugins[0]
module_path = plugin.frontend_metadata().es_module_path
if not module_path:
return http_util.Respond(
request, "Plugin is not module loadable", "text/plain", code=400
)
# non-self origin is blocked by CSP but this is a good invariant checking.
if urlparse.urlparse(module_path).netloc:
raise ValueError("Expected es_module_path to be non-absolute path")
module_json = json.dumps("." + module_path)
script_content = "import({}).then((m) => void m.render());".format(
module_json
)
digest = hashlib.sha256(script_content.encode("utf-8")).digest()
script_sha = base64.b64encode(digest).decode("ascii")
html = textwrap.dedent(
"""
<!DOCTYPE html>
<head><base href="plugin/{name}/" /></head>
<body><script type="module">{script_content}</script></body>
"""
).format(name=name, script_content=script_content)
return http_util.Respond(
request,
html,
"text/html",
csp_scripts_sha256s=[script_sha],
)
@wrappers.Request.application
def _serve_plugins_listing(self, request):
"""Serves an object mapping plugin name to whether it is enabled.
Args:
request: The werkzeug.Request object.
Returns:
A werkzeug.Response object.
"""
response = collections.OrderedDict()
ctx = plugin_util.context(request.environ)
eid = plugin_util.experiment_id(request.environ)
plugins_with_data = frozenset(
self._data_provider.list_plugins(ctx, experiment_id=eid)
or frozenset()
if self._data_provider is not None
else frozenset()
)
plugins_to_skip = self._experimental_plugins - frozenset(
request.args.getlist(EXPERIMENTAL_PLUGINS_QUERY_PARAM)
)
for plugin in self._plugins:
if plugin.plugin_name in plugins_to_skip:
continue
if (
type(plugin) is core_plugin.CorePlugin
): # pylint: disable=unidiomatic-typecheck
# This plugin's existence is a backend implementation detail.
continue
is_active = bool(
frozenset(plugin.data_plugin_names()) & plugins_with_data
)
if not is_active:
try:
start = time.time()
is_active = plugin.is_active()
elapsed = time.time() - start
logger.info(
"Plugin listing: is_active() for %s took %0.3f seconds",
plugin.plugin_name,
elapsed,
)
except Exception:
is_active = False
logger.error(
"Plugin listing: is_active() for %s failed (marking inactive)",
plugin.plugin_name,
exc_info=True,
)
plugin_metadata = plugin.frontend_metadata()
output_metadata = {
"disable_reload": plugin_metadata.disable_reload,
"enabled": is_active,
# loading_mechanism set below
"remove_dom": plugin_metadata.remove_dom,
# tab_name set below
}
if plugin_metadata.tab_name is not None:
output_metadata["tab_name"] = plugin_metadata.tab_name
else:
output_metadata["tab_name"] = plugin.plugin_name
es_module_handler = plugin_metadata.es_module_path
element_name = plugin_metadata.element_name
is_ng_component = plugin_metadata.is_ng_component
if is_ng_component:
if element_name is not None:
raise ValueError(
"Plugin %r declared as both Angular built-in and legacy"
% plugin.plugin_name
)
if es_module_handler is not None:
raise ValueError(
"Plugin %r declared as both Angular built-in and iframed"
% plugin.plugin_name
)
loading_mechanism = {
"type": "NG_COMPONENT",
}
elif element_name is not None and es_module_handler is not None:
logger.error(
"Plugin %r declared as both legacy and iframed; skipping",
plugin.plugin_name,
)
continue
elif element_name is not None and es_module_handler is None:
loading_mechanism = {
"type": "CUSTOM_ELEMENT",
"element_name": element_name,
}
elif element_name is None and es_module_handler is not None:
loading_mechanism = {
"type": "IFRAME",
"module_path": "".join(
[
request.script_root,
DATA_PREFIX,
PLUGIN_PREFIX,
"/",
plugin.plugin_name,
es_module_handler,
]
),
}
else:
# As a compatibility measure (for plugins that we don't
# control), we'll pull it from the frontend registry for now.
loading_mechanism = {
"type": "NONE",
}
output_metadata["loading_mechanism"] = loading_mechanism
response[plugin.plugin_name] = output_metadata
return http_util.Respond(request, response, "application/json")
def __call__(self, environ, start_response):
"""Central entry point for the TensorBoard application.
This __call__ method conforms to the WSGI spec, so that instances of this
class are WSGI applications.
Args:
environ: See WSGI spec (PEP 3333).
start_response: See WSGI spec (PEP 3333).
"""
return self._app(environ, start_response)
def _route_request(self, environ, start_response):
"""Delegate an incoming request to sub-applications.
This method supports strict string matching and wildcard routes of a
single path component, such as `/foo/*`. Other routing patterns,
like regular expressions, are not supported.
This is the main TensorBoard entry point before middleware is
applied. (See `_create_wsgi_app`.)
Args:
environ: See WSGI spec (PEP 3333).
start_response: See WSGI spec (PEP 3333).
"""
request = wrappers.Request(environ)
parsed_url = urlparse.urlparse(request.path)
clean_path = _clean_path(parsed_url.path)
# pylint: disable=too-many-function-args
if clean_path in self.exact_routes:
return self.exact_routes[clean_path](environ, start_response)
else:
for path_prefix in self.prefix_routes:
if clean_path.startswith(path_prefix):
return self.prefix_routes[path_prefix](
environ, start_response
)
logger.warning("path %s not found, sending 404", clean_path)
return http_util.Respond(
request, "Not found", "text/plain", code=404
)(environ, start_response)
# pylint: enable=too-many-function-args
def _handling_errors(wsgi_app):
def wrapper(environ, start_response):
try:
return wsgi_app(environ, start_response)
except errors.PublicError as e:
request = wrappers.Request(environ)
error_app = http_util.Respond(
request,
str(e),
"text/plain",
code=e.http_code,
headers=e.headers,
)
return error_app(environ, start_response)
# Let other exceptions be handled by the server, as an opaque
# internal server error.
return wrapper
def _clean_path(path):
"""Removes a trailing slash from a non-root path.
Arguments:
path: The path of a request.
Returns:
The route to use to serve the request.
"""
if path != "/" and path.endswith("/"):
return path[:-1]
return path
def _placeholder_assets_zip_provider():
"""Defines a default assets zip provider containing a dummy index.html."""
memfile = io.BytesIO()
with zipfile.ZipFile(memfile, mode="w") as zf:
zf.writestr("index.html", "TensorBoard placeholder index.html")
return memfile
@@ -0,0 +1,38 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from tensorboard import auth
from tensorboard import context
class AuthContextMiddleware:
"""WSGI middleware to inject an AuthContext into the RequestContext."""
def __init__(self, application, auth_providers):
"""Initializes this middleware.
Args:
application: A WSGI application to delegate to.
auth_providers: The auth_providers to provide to the AuthContext.
"""
self._application = application
self._auth_providers = auth_providers
def __call__(self, environ, start_response):
"""Invoke this WSGI application."""
environ = dict(environ)
auth_ctx = auth.AuthContext(self._auth_providers, environ)
ctx = context.from_environ(environ).replace(auth=auth_ctx)
context.set_in_environ(environ, ctx)
return self._application(environ, start_response)
@@ -0,0 +1,113 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Middleware for injecting client-side feature flags into the Context."""
import json
import urllib.parse
from tensorboard import context
from tensorboard import errors
class ClientFeatureFlagsMiddleware:
"""Middleware for injecting client-side feature flags into the Context.
The client webapp is expected to include a json-serialized version of its
FeatureFlags in the `X-TensorBoard-Feature-Flags` header or the
`tensorBoardFeatureFlags` query parameter. This middleware extracts the
header or query parameter value and converts it into the client_feature_flags
property for the DataProvider's Context object, where client_feature_flags
is a Dict of string keys and arbitrary value types.
In the event that both the header and query parameter are specified, the
values from the header will take precedence.
"""
def __init__(self, application):
"""Initializes this middleware.
Args:
application: The WSGI application to wrap (see PEP 3333).
"""
self._application = application
def __call__(self, environ, start_response):
header_feature_flags = self._parse_potential_header_param_flags(
environ.get("HTTP_X_TENSORBOARD_FEATURE_FLAGS")
)
query_string_feature_flags = self._parse_potential_query_param_flags(
environ.get("QUERY_STRING")
)
if not header_feature_flags and not query_string_feature_flags:
return self._application(environ, start_response)
# header flags take precedence
for flag, value in header_feature_flags.items():
query_string_feature_flags[flag] = value
ctx = context.from_environ(environ).replace(
client_feature_flags=query_string_feature_flags
)
context.set_in_environ(environ, ctx)
return self._application(environ, start_response)
def _parse_potential_header_param_flags(self, header_string):
if not header_string:
return {}
try:
header_feature_flags = json.loads(header_string)
except json.JSONDecodeError:
raise errors.InvalidArgumentError(
"X-TensorBoard-Feature-Flags cannot be JSON decoded."
)
if not isinstance(header_feature_flags, dict):
raise errors.InvalidArgumentError(
"X-TensorBoard-Feature-Flags cannot be decoded to a dict."
)
return header_feature_flags
def _parse_potential_query_param_flags(self, query_string):
if not query_string:
return {}
try:
query_string_json = urllib.parse.parse_qs(query_string)
except ValueError:
return {}
# parse_qs returns the dictionary values as lists for each name.
potential_feature_flags = query_string_json.get(
"tensorBoardFeatureFlags", []
)
if not potential_feature_flags:
return {}
try:
client_feature_flags = json.loads(potential_feature_flags[0])
except json.JSONDecodeError:
raise errors.InvalidArgumentError(
"tensorBoardFeatureFlags cannot be JSON decoded."
)
if not isinstance(client_feature_flags, dict):
raise errors.InvalidArgumentError(
"tensorBoardFeatureFlags cannot be decoded to a dict."
)
return client_feature_flags
@@ -0,0 +1,46 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Redirect from an empty path to the virtual application root.
Sometimes, middleware transformations will make the path empty: for
example, navigating to "/foo" (no trailing slash) when the path prefix
is exactly "/foo". In such cases, relative links on the frontend would
break. Instead of handling this special case in each relevant
middleware, we install a top-level redirect handler from "" to "/".
This middleware respects `SCRIPT_NAME` as described by the WSGI spec. If
`SCRIPT_NAME` is set to "/foo", then an empty `PATH_INFO` corresponds to
the actual path "/foo", and so will be redirected to "/foo/".
"""
class EmptyPathRedirectMiddleware:
"""WSGI middleware to redirect from "" to "/"."""
def __init__(self, application):
"""Initializes this middleware.
Args:
application: The WSGI application to wrap (see PEP 3333).
"""
self._application = application
def __call__(self, environ, start_response):
path = environ.get("PATH_INFO", "")
if path:
return self._application(environ, start_response)
location = environ.get("SCRIPT_NAME", "") + "/"
start_response("301 Moved Permanently", [("Location", location)])
return []
@@ -0,0 +1,277 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides data ingestion logic backed by local event processing."""
import os
import re
import threading
import time
from tensorboard.backend.event_processing import data_provider
from tensorboard.backend.event_processing import plugin_event_multiplexer
from tensorboard.backend.event_processing import tag_types
from tensorboard.compat import tf
from tensorboard.data import ingester
from tensorboard.plugins.audio import metadata as audio_metadata
from tensorboard.plugins.histogram import metadata as histogram_metadata
from tensorboard.plugins.image import metadata as image_metadata
from tensorboard.plugins.pr_curve import metadata as pr_curve_metadata
from tensorboard.plugins.scalar import metadata as scalar_metadata
from tensorboard.util import tb_logging
DEFAULT_SIZE_GUIDANCE = {
tag_types.TENSORS: 10,
}
# TODO(@wchargin): Replace with something that works for third-party plugins.
DEFAULT_TENSOR_SIZE_GUIDANCE = {
scalar_metadata.PLUGIN_NAME: 1000,
image_metadata.PLUGIN_NAME: 10,
audio_metadata.PLUGIN_NAME: 10,
histogram_metadata.PLUGIN_NAME: 500,
pr_curve_metadata.PLUGIN_NAME: 100,
}
logger = tb_logging.get_logger()
class LocalDataIngester(ingester.DataIngester):
"""Data ingestion implementation to use when running locally."""
def __init__(self, flags):
"""Initializes a `LocalDataIngester` from `flags`.
Args:
flags: An argparse.Namespace containing TensorBoard CLI flags.
Returns:
The new `LocalDataIngester`.
"""
tensor_size_guidance = dict(DEFAULT_TENSOR_SIZE_GUIDANCE)
tensor_size_guidance.update(flags.samples_per_plugin)
self._multiplexer = plugin_event_multiplexer.EventMultiplexer(
size_guidance=DEFAULT_SIZE_GUIDANCE,
tensor_size_guidance=tensor_size_guidance,
purge_orphaned_data=flags.purge_orphaned_data,
max_reload_threads=flags.max_reload_threads,
event_file_active_filter=_get_event_file_active_filter(flags),
detect_file_replacement=flags.detect_file_replacement,
)
self._data_provider = data_provider.MultiplexerDataProvider(
self._multiplexer, flags.logdir or flags.logdir_spec
)
self._reload_interval = flags.reload_interval
self._reload_task = flags.reload_task
if flags.logdir:
self._path_to_run = {os.path.expanduser(flags.logdir): None}
else:
self._path_to_run = _parse_event_files_spec(flags.logdir_spec)
# Conditionally import tensorflow_io.
if getattr(tf, "__version__", "stub") != "stub":
_check_filesystem_support(self._path_to_run.keys())
@property
def data_provider(self):
return self._data_provider
@property
def deprecated_multiplexer(self):
return self._multiplexer
def start(self):
"""Starts ingesting data based on the ingester flag configuration."""
def _reload():
while True:
start = time.time()
logger.info("TensorBoard reload process beginning")
for path, name in self._path_to_run.items():
self._multiplexer.AddRunsFromDirectory(path, name)
logger.info(
"TensorBoard reload process: Reload the whole Multiplexer"
)
self._multiplexer.Reload()
duration = time.time() - start
logger.info(
"TensorBoard done reloading. Load took %0.3f secs", duration
)
if self._reload_interval == 0:
# Only load the multiplexer once. Do not continuously reload.
break
time.sleep(self._reload_interval)
if self._reload_task == "process":
logger.info("Launching reload in a child process")
import multiprocessing
process = multiprocessing.Process(target=_reload, name="Reloader")
# Best-effort cleanup; on exit, the main TB parent process will attempt to
# kill all its daemonic children.
process.daemon = True
process.start()
elif self._reload_task in ("thread", "auto"):
logger.info("Launching reload in a daemon thread")
thread = threading.Thread(target=_reload, name="Reloader")
# Make this a daemon thread, which won't block TB from exiting.
thread.daemon = True
thread.start()
elif self._reload_task == "blocking":
if self._reload_interval != 0:
raise ValueError(
"blocking reload only allowed with load_interval=0"
)
_reload()
else:
raise ValueError("unrecognized reload_task: %s" % self._reload_task)
def _get_event_file_active_filter(flags):
"""Returns a predicate for whether an event file load timestamp is active.
Returns:
A predicate function accepting a single UNIX timestamp float argument, or
None if multi-file loading is not enabled.
"""
if not flags.reload_multifile:
return None
inactive_secs = flags.reload_multifile_inactive_secs
if inactive_secs == 0:
return None
if inactive_secs < 0:
return lambda timestamp: True
return lambda timestamp: timestamp + inactive_secs >= time.time()
def _parse_event_files_spec(logdir_spec):
"""Parses `logdir_spec` into a map from paths to run group names.
The `--logdir_spec` flag format is a comma-separated list of path
specifications. A path spec looks like 'group_name:/path/to/directory' or
'/path/to/directory'; in the latter case, the group is unnamed. Group names
cannot start with a forward slash: /foo:bar/baz will be interpreted as a spec
with no name and path '/foo:bar/baz'.
Globs are not supported.
Args:
logdir: A comma-separated list of run specifications.
Returns:
A dict mapping directory paths to names like {'/path/to/directory': 'name'}.
Groups without an explicit name are named after their path. If logdir is
None, returns an empty dict, which is helpful for testing things that don't
require any valid runs.
"""
files = {}
if logdir_spec is None:
return files
# Make sure keeping consistent with ParseURI in core/lib/io/path.cc
uri_pattern = re.compile("[a-zA-Z][0-9a-zA-Z.]*://.*")
for specification in logdir_spec.split(","):
# Check if the spec contains group. A spec start with xyz:// is regarded as
# URI path spec instead of group spec. If the spec looks like /foo:bar/baz,
# then we assume it's a path with a colon. If the spec looks like
# [a-zA-z]:\foo then we assume its a Windows path and not a single letter
# group
if (
uri_pattern.match(specification) is None
and ":" in specification
and specification[0] != "/"
and not os.path.splitdrive(specification)[0]
):
# We split at most once so run_name:/path:with/a/colon will work.
run_name, _, path = specification.partition(":")
else:
run_name = None
path = specification
if uri_pattern.match(path) is None:
path = os.path.realpath(os.path.expanduser(path))
files[path] = run_name
return files
def _get_filesystem_scheme(path):
"""Extracts filesystem scheme from a given path.
The filesystem scheme is usually separated by `://` from the local filesystem
path if given. For example, the scheme of `file://tmp/tf` is `file`.
Args:
path: A strings representing an input log directory.
Returns:
Filesystem scheme, None if the path doesn't contain one.
"""
if "://" not in path:
return None
return path.split("://")[0]
def _check_filesystem_support(paths):
"""Examines the list of filesystems user requested.
If TF I/O schemes are requested, try to import tensorflow_io module.
Args:
paths: A list of strings representing input log directories.
"""
get_registered_schemes = getattr(
tf.io.gfile, "get_registered_schemes", None
)
registered_schemes = (
None if get_registered_schemes is None else get_registered_schemes()
)
# Only need to check one path for each scheme.
scheme_to_path = {_get_filesystem_scheme(path): path for path in paths}
missing_scheme = None
for scheme, path in scheme_to_path.items():
if scheme is None:
continue
# Use `tf.io.gfile.exists.get_registered_schemes` if possible.
if registered_schemes is not None:
if scheme not in registered_schemes:
missing_scheme = scheme
break
else:
# Fall back to `tf.io.gfile.exists`.
try:
tf.io.gfile.exists(path)
except tf.errors.UnimplementedError:
missing_scheme = scheme
break
except tf.errors.OpError:
# Swallow other errors; we aren't concerned about them at this point.
pass
if missing_scheme:
try:
import tensorflow_io # noqa: F401
except ImportError as e:
supported_schemes_msg = (
" (supported schemes: {})".format(registered_schemes)
if registered_schemes
else ""
)
raise tf.errors.UnimplementedError(
None,
None,
(
"Error: Unsupported filename scheme '{}'{}. For additional"
+ " filesystem support, consider installing TensorFlow I/O"
+ " (https://www.tensorflow.org/io) via `pip install tensorflow-io`."
).format(missing_scheme, supported_schemes_msg),
) from e
@@ -0,0 +1,538 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Bridge from event multiplexer storage to generic data APIs."""
import base64
import collections
import json
import random
from tensorboard import errors
from tensorboard.compat.proto import summary_pb2
from tensorboard.data import provider
from tensorboard.util import tb_logging
from tensorboard.util import tensor_util
logger = tb_logging.get_logger()
class MultiplexerDataProvider(provider.DataProvider):
def __init__(self, multiplexer, logdir):
"""Trivial initializer.
Args:
multiplexer: A `plugin_event_multiplexer.EventMultiplexer` (note:
not a boring old `event_multiplexer.EventMultiplexer`).
logdir: The log directory from which data is being read. Only used
cosmetically. Should be a `str`.
"""
self._multiplexer = multiplexer
self._logdir = logdir
def __str__(self):
return "MultiplexerDataProvider(logdir=%r)" % self._logdir
def _validate_context(self, ctx):
if type(ctx).__name__ != "RequestContext":
raise TypeError("ctx must be a RequestContext; got: %r" % (ctx,))
def _validate_experiment_id(self, experiment_id):
# This data provider doesn't consume the experiment ID at all, but
# as a courtesy to callers we require that it be a valid string, to
# help catch usage errors.
if not isinstance(experiment_id, str):
raise TypeError(
"experiment_id must be %r, but got %r: %r"
% (str, type(experiment_id), experiment_id)
)
def _validate_downsample(self, downsample):
if downsample is None:
raise TypeError("`downsample` required but not given")
if isinstance(downsample, int):
return # OK
raise TypeError(
"`downsample` must be an int, but got %r: %r"
% (type(downsample), downsample)
)
def _test_run_tag(self, run_tag_filter, run, tag):
runs = run_tag_filter.runs
if runs is not None and run not in runs:
return False
tags = run_tag_filter.tags
if tags is not None and tag not in tags:
return False
return True
def _get_first_event_timestamp(self, run_name):
try:
return self._multiplexer.FirstEventTimestamp(run_name)
except ValueError as e:
return None
def experiment_metadata(self, ctx=None, *, experiment_id):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
return provider.ExperimentMetadata(data_location=self._logdir)
def list_plugins(self, ctx=None, *, experiment_id):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
# Note: This result may include plugins that only have time
# series with `DATA_CLASS_UNKNOWN`, which will not actually be
# accessible via `list_*` or read_*`. This is inconsistent with
# the specification for `list_plugins`, but the bug should be
# mostly harmless.
return self._multiplexer.ActivePlugins()
def list_runs(self, ctx=None, *, experiment_id):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
return [
provider.Run(
run_id=run, # use names as IDs
run_name=run,
start_time=self._get_first_event_timestamp(run),
)
for run in self._multiplexer.Runs()
]
def list_scalars(
self, ctx=None, *, experiment_id, plugin_name, run_tag_filter=None
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_SCALAR
)
return self._list(provider.ScalarTimeSeries, index)
def read_scalars(
self,
ctx=None,
*,
experiment_id,
plugin_name,
downsample=None,
run_tag_filter=None,
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
self._validate_downsample(downsample)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_SCALAR
)
return self._read(_convert_scalar_event, index, downsample)
def read_last_scalars(
self,
ctx=None,
*,
experiment_id,
plugin_name,
run_tag_filter=None,
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_SCALAR
)
run_tag_to_last_scalar_datum = collections.defaultdict(dict)
for run, tags_for_run in index.items():
for tag, metadata in tags_for_run.items():
events = self._multiplexer.Tensors(run, tag)
if events:
run_tag_to_last_scalar_datum[run][tag] = (
_convert_scalar_event(events[-1])
)
return run_tag_to_last_scalar_datum
def list_tensors(
self, ctx=None, *, experiment_id, plugin_name, run_tag_filter=None
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_TENSOR
)
return self._list(provider.TensorTimeSeries, index)
def read_tensors(
self,
ctx=None,
*,
experiment_id,
plugin_name,
downsample=None,
run_tag_filter=None,
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
self._validate_downsample(downsample)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_TENSOR
)
return self._read(_convert_tensor_event, index, downsample)
def _index(self, plugin_name, run_tag_filter, data_class_filter):
"""List time series and metadata matching the given filters.
This is like `_list`, but doesn't traverse `Tensors(...)` to
compute metadata that's not always needed.
Args:
plugin_name: A string plugin name filter (required).
run_tag_filter: An `provider.RunTagFilter`, or `None`.
data_class_filter: A `summary_pb2.DataClass` filter (required).
Returns:
A nested dict `d` such that `d[run][tag]` is a
`SummaryMetadata` proto.
"""
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter(runs=None, tags=None)
runs = run_tag_filter.runs
tags = run_tag_filter.tags
# Optimization for a common case, reading a single time series.
if runs and len(runs) == 1 and tags and len(tags) == 1:
(run,) = runs
(tag,) = tags
try:
metadata = self._multiplexer.SummaryMetadata(run, tag)
except KeyError:
return {}
all_metadata = {run: {tag: metadata}}
else:
all_metadata = self._multiplexer.AllSummaryMetadata()
result = {}
for run, tag_to_metadata in all_metadata.items():
if runs is not None and run not in runs:
continue
result_for_run = {}
for tag, metadata in tag_to_metadata.items():
if tags is not None and tag not in tags:
continue
if metadata.data_class != data_class_filter:
continue
if metadata.plugin_data.plugin_name != plugin_name:
continue
result[run] = result_for_run
result_for_run[tag] = metadata
return result
def _list(self, construct_time_series, index):
"""Helper to list scalar or tensor time series.
Args:
construct_time_series: `ScalarTimeSeries` or `TensorTimeSeries`.
index: The result of `self._index(...)`.
Returns:
A list of objects of type given by `construct_time_series`,
suitable to be returned from `list_scalars` or `list_tensors`.
"""
result = {}
for run, tag_to_metadata in index.items():
result_for_run = {}
result[run] = result_for_run
for tag, summary_metadata in tag_to_metadata.items():
max_step = None
max_wall_time = None
for event in self._multiplexer.Tensors(run, tag):
if max_step is None or max_step < event.step:
max_step = event.step
if max_wall_time is None or max_wall_time < event.wall_time:
max_wall_time = event.wall_time
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
result_for_run[tag] = construct_time_series(
max_step=max_step,
max_wall_time=max_wall_time,
plugin_content=summary_metadata.plugin_data.content,
description=summary_metadata.summary_description,
display_name=summary_metadata.display_name,
)
return result
def _read(self, convert_event, index, downsample):
"""Helper to read scalar or tensor data from the multiplexer.
Args:
convert_event: Takes `plugin_event_accumulator.TensorEvent` to
either `provider.ScalarDatum` or `provider.TensorDatum`.
index: The result of `self._index(...)`.
downsample: Non-negative `int`; how many samples to return per
time series.
Returns:
A dict of dicts of values returned by `convert_event` calls,
suitable to be returned from `read_scalars` or `read_tensors`.
"""
result = {}
for run, tags_for_run in index.items():
result_for_run = {}
result[run] = result_for_run
for tag, metadata in tags_for_run.items():
events = self._multiplexer.Tensors(run, tag)
data = [convert_event(e) for e in events]
result_for_run[tag] = _downsample(data, downsample)
return result
def list_blob_sequences(
self, ctx=None, *, experiment_id, plugin_name, run_tag_filter=None
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_BLOB_SEQUENCE
)
result = {}
for run, tag_to_metadata in index.items():
result_for_run = {}
result[run] = result_for_run
for tag, metadata in tag_to_metadata.items():
max_step = None
max_wall_time = None
max_length = None
for event in self._multiplexer.Tensors(run, tag):
if max_step is None or max_step < event.step:
max_step = event.step
if max_wall_time is None or max_wall_time < event.wall_time:
max_wall_time = event.wall_time
length = _tensor_size(event.tensor_proto)
if max_length is None or length > max_length:
max_length = length
result_for_run[tag] = provider.BlobSequenceTimeSeries(
max_step=max_step,
max_wall_time=max_wall_time,
max_length=max_length,
plugin_content=metadata.plugin_data.content,
description=metadata.summary_description,
display_name=metadata.display_name,
)
return result
def read_blob_sequences(
self,
ctx=None,
*,
experiment_id,
plugin_name,
downsample=None,
run_tag_filter=None,
):
self._validate_context(ctx)
self._validate_experiment_id(experiment_id)
self._validate_downsample(downsample)
index = self._index(
plugin_name, run_tag_filter, summary_pb2.DATA_CLASS_BLOB_SEQUENCE
)
result = {}
for run, tags in index.items():
result_for_run = {}
result[run] = result_for_run
for tag in tags:
events = self._multiplexer.Tensors(run, tag)
data_by_step = {}
for event in events:
if event.step in data_by_step:
continue
data_by_step[event.step] = _convert_blob_sequence_event(
experiment_id, plugin_name, run, tag, event
)
data = [datum for (step, datum) in sorted(data_by_step.items())]
result_for_run[tag] = _downsample(data, downsample)
return result
def read_blob(self, ctx=None, *, blob_key):
self._validate_context(ctx)
(
unused_experiment_id,
plugin_name,
run,
tag,
step,
index,
) = _decode_blob_key(blob_key)
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
if summary_metadata.data_class != summary_pb2.DATA_CLASS_BLOB_SEQUENCE:
raise errors.NotFoundError(blob_key)
tensor_events = self._multiplexer.Tensors(run, tag)
# In case of multiple events at this step, take first (arbitrary).
matching_step = next((e for e in tensor_events if e.step == step), None)
if not matching_step:
raise errors.NotFoundError("%s: no such step %r" % (blob_key, step))
tensor = tensor_util.make_ndarray(matching_step.tensor_proto)
return tensor[index]
# TODO(davidsoergel): deduplicate with other implementations
def _encode_blob_key(experiment_id, plugin_name, run, tag, step, index):
"""Generate a blob key: a short, URL-safe string identifying a blob.
A blob can be located using a set of integer and string fields; here we
serialize these to allow passing the data through a URL. Specifically, we
1) construct a tuple of the arguments in order; 2) represent that as an
ascii-encoded JSON string (without whitespace); and 3) take the URL-safe
base64 encoding of that, with no padding. For example:
1) Tuple: ("some_id", "graphs", "train", "graph_def", 2, 0)
2) JSON: ["some_id","graphs","train","graph_def",2,0]
3) base64: WyJzb21lX2lkIiwiZ3JhcGhzIiwidHJhaW4iLCJncmFwaF9kZWYiLDIsMF0K
Args:
experiment_id: a string ID identifying an experiment.
plugin_name: string
run: string
tag: string
step: int
index: int
Returns:
A URL-safe base64-encoded string representing the provided arguments.
"""
# Encodes the blob key as a URL-safe string, as required by the
# `BlobReference` API in `tensorboard/data/provider.py`, because these keys
# may be used to construct URLs for retrieving blobs.
stringified = json.dumps(
(experiment_id, plugin_name, run, tag, step, index),
separators=(",", ":"),
)
bytesified = stringified.encode("ascii")
encoded = base64.urlsafe_b64encode(bytesified)
return encoded.decode("ascii").rstrip("=")
# Any changes to this function need not be backward-compatible, even though
# the current encoding was used to generate URLs. The reason is that the
# generated URLs are not considered permalinks: they need to be valid only
# within the context of the session that created them (via the matching
# `_encode_blob_key` function above).
def _decode_blob_key(key):
"""Decode a blob key produced by `_encode_blob_key` into component fields.
Args:
key: a blob key, as generated by `_encode_blob_key`.
Returns:
A tuple of `(experiment_id, plugin_name, run, tag, step, index)`, with types
matching the arguments of `_encode_blob_key`.
"""
decoded = base64.urlsafe_b64decode(key + "==") # pad past a multiple of 4.
stringified = decoded.decode("ascii")
(experiment_id, plugin_name, run, tag, step, index) = json.loads(
stringified
)
return (experiment_id, plugin_name, run, tag, step, index)
def _convert_scalar_event(event):
"""Helper for `read_scalars`."""
return provider.ScalarDatum(
step=event.step,
wall_time=event.wall_time,
value=tensor_util.make_ndarray(event.tensor_proto).item(),
)
def _convert_tensor_event(event):
"""Helper for `read_tensors`."""
return provider.TensorDatum(
step=event.step,
wall_time=event.wall_time,
numpy=tensor_util.make_ndarray(event.tensor_proto),
)
def _convert_blob_sequence_event(experiment_id, plugin_name, run, tag, event):
"""Helper for `read_blob_sequences`."""
num_blobs = _tensor_size(event.tensor_proto)
values = tuple(
provider.BlobReference(
_encode_blob_key(
experiment_id,
plugin_name,
run,
tag,
event.step,
idx,
)
)
for idx in range(num_blobs)
)
return provider.BlobSequenceDatum(
wall_time=event.wall_time,
step=event.step,
values=values,
)
def _tensor_size(tensor_proto):
"""Compute the number of elements in a tensor.
This does not deserialize the full tensor contents.
Args:
tensor_proto: A `tensorboard.compat.proto.tensor_pb2.TensorProto`.
Returns:
A non-negative `int`.
"""
# This is the same logic that `tensor_util.make_ndarray` uses to
# compute the size, but without the actual buffer copies.
result = 1
for dim in tensor_proto.tensor_shape.dim:
result *= dim.size
return result
def _downsample(xs, k):
"""Downsample `xs` to at most `k` elements.
If `k` is larger than `xs`, then the contents of `xs` itself will be
returned. If `k` is smaller than `xs`, the last element of `xs` will
always be included (unless `k` is `0`) and the preceding elements
will be selected uniformly at random.
This differs from `random.sample` in that it returns a subsequence
(i.e., order is preserved) and that it permits `k > len(xs)`.
The random number generator will always be `random.Random(0)`, so
this function is deterministic (within a Python process).
Args:
xs: A sequence (`collections.abc.Sequence`).
k: A non-negative integer.
Returns:
A new list whose elements are a subsequence of `xs` of length
`min(k, len(xs))` and that is guaranteed to include the last
element of `xs`, uniformly selected among such subsequences.
"""
if k > len(xs):
return list(xs)
if k == 0:
return []
indices = random.Random(0).sample(range(len(xs) - 1), k - 1)
indices.sort()
indices += [len(xs) - 1]
return [xs[i] for i in indices]
@@ -0,0 +1,144 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Implementation for a multi-file directory loader."""
from tensorboard.backend.event_processing import directory_watcher
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.compat import tf
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
# Sentinel object for an inactive path.
_INACTIVE = object()
class DirectoryLoader:
"""Loader for an entire directory, maintaining multiple active file
loaders.
This class takes a directory, a factory for loaders, and optionally a
path filter and watches all the paths inside that directory for new data.
Each file loader created by the factory must read a path and produce an
iterator of (timestamp, value) pairs.
Unlike DirectoryWatcher, this class does not assume that only one file
receives new data at a time; there can be arbitrarily many active files.
However, any file whose maximum load timestamp fails an "active" predicate
will be marked as inactive and no longer checked for new data.
"""
def __init__(
self,
directory,
loader_factory,
path_filter=lambda x: True,
active_filter=lambda timestamp: True,
):
"""Constructs a new MultiFileDirectoryLoader.
Args:
directory: The directory to load files from.
loader_factory: A factory for creating loaders. The factory should take a
path and return an object that has a Load method returning an iterator
yielding (unix timestamp as float, value) pairs for any new data
path_filter: If specified, only paths matching this filter are loaded.
active_filter: If specified, any loader whose maximum load timestamp does
not pass this filter will be marked as inactive and no longer read.
Raises:
ValueError: If directory or loader_factory are None.
"""
if directory is None:
raise ValueError("A directory is required")
if loader_factory is None:
raise ValueError("A loader factory is required")
self._directory = directory
self._loader_factory = loader_factory
self._path_filter = path_filter
self._active_filter = active_filter
self._loaders = {}
self._max_timestamps = {}
def Load(self):
"""Loads new values from all active files.
Yields:
All values that have not been yielded yet.
Raises:
DirectoryDeletedError: If the directory has been permanently deleted
(as opposed to being temporarily unavailable).
"""
try:
all_paths = io_wrapper.ListDirectoryAbsolute(self._directory)
paths = sorted(p for p in all_paths if self._path_filter(p))
for path in paths:
for value in self._LoadPath(path):
yield value
except tf.errors.OpError as e:
if not tf.io.gfile.exists(self._directory):
raise directory_watcher.DirectoryDeletedError(
"Directory %s has been permanently deleted"
% self._directory
)
else:
logger.info("Ignoring error during file loading: %s" % e)
def _LoadPath(self, path):
"""Generator for values from a single path's loader.
Args:
path: the path to load from
Yields:
All values from this path's loader that have not been yielded yet.
"""
max_timestamp = self._max_timestamps.get(path, None)
if max_timestamp is _INACTIVE or self._MarkIfInactive(
path, max_timestamp
):
logger.debug("Skipping inactive path %s", path)
return
loader = self._loaders.get(path, None)
if loader is None:
try:
loader = self._loader_factory(path)
except tf.errors.NotFoundError:
# Happens if a file was removed after we listed the directory.
logger.debug("Skipping nonexistent path %s", path)
return
self._loaders[path] = loader
logger.info("Loading data from path %s", path)
for timestamp, value in loader.Load():
if max_timestamp is None or timestamp > max_timestamp:
max_timestamp = timestamp
yield value
if not self._MarkIfInactive(path, max_timestamp):
self._max_timestamps[path] = max_timestamp
def _MarkIfInactive(self, path, max_timestamp):
"""If max_timestamp is inactive, returns True and marks the path as
such."""
logger.debug("Checking active status of %s at %s", path, max_timestamp)
if max_timestamp is not None and not self._active_filter(max_timestamp):
self._max_timestamps[path] = _INACTIVE
del self._loaders[path]
return True
return False
@@ -0,0 +1,273 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Contains the implementation for the DirectoryWatcher class."""
import bisect
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.compat import tf
from tensorboard.util import io_util
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
class DirectoryWatcher:
"""A DirectoryWatcher wraps a loader to load from a sequence of paths.
A loader reads a path and produces some kind of values as an iterator. A
DirectoryWatcher takes a directory, a factory for loaders, and optionally a
path filter and watches all the paths inside that directory.
This class is only valid under the assumption that only one path will be
written to by the data source at a time and that once the source stops writing
to a path, it will start writing to a new path that's lexicographically
greater and never come back. It uses some heuristics to check whether this is
true based on tracking changes to the files' sizes, but the check can have
false negatives. However, it should have no false positives.
"""
def __init__(self, directory, loader_factory, path_filter=lambda x: True):
"""Constructs a new DirectoryWatcher.
Args:
directory: The directory to load files from.
loader_factory: A factory for creating loaders. The factory should take a
path and return an object that has a Load method returning an
iterator that will yield all events that have not been yielded yet.
path_filter: If specified, only paths matching this filter are loaded.
Raises:
ValueError: If path_provider or loader_factory are None.
"""
if directory is None:
raise ValueError("A directory is required")
if loader_factory is None:
raise ValueError("A loader factory is required")
self._directory = directory
self._path = None
self._loader_factory = loader_factory
self._loader = None
self._path_filter = path_filter
self._ooo_writes_detected = False
# The file size for each file at the time it was finalized.
self._finalized_sizes = {}
def Load(self):
"""Loads new values.
The watcher will load from one path at a time; as soon as that path stops
yielding events, it will move on to the next path. We assume that old paths
are never modified after a newer path has been written. As a result, Load()
can be called multiple times in a row without losing events that have not
been yielded yet. In other words, we guarantee that every event will be
yielded exactly once.
Yields:
All values that have not been yielded yet.
Raises:
DirectoryDeletedError: If the directory has been permanently deleted
(as opposed to being temporarily unavailable).
"""
try:
for event in self._LoadInternal():
yield event
except tf.errors.OpError:
if not tf.io.gfile.exists(self._directory):
raise DirectoryDeletedError(
"Directory %s has been permanently deleted"
% self._directory
)
def _LoadInternal(self):
"""Internal implementation of Load().
The only difference between this and Load() is that the latter will throw
DirectoryDeletedError on I/O errors if it thinks that the directory has been
permanently deleted.
Yields:
All values that have not been yielded yet.
"""
# If the loader exists, check it for a value.
if not self._loader:
self._InitializeLoader()
# If it still doesn't exist, there is no data
if not self._loader:
return
while True:
# Yield all the new events in the path we're currently loading from.
for event in self._loader.Load():
yield event
next_path = self._GetNextPath()
if not next_path:
logger.info("No path found after %s", self._path)
# Current path is empty and there are no new paths, so we're done.
return
# There's a new path, so check to make sure there weren't any events
# written between when we finished reading the current path and when we
# checked for the new one. The sequence of events might look something
# like this:
#
# 1. Event #1 written to path #1.
# 2. We check for events and yield event #1 from path #1
# 3. We check for events and see that there are no more events in path #1.
# 4. Event #2 is written to path #1.
# 5. Event #3 is written to path #2.
# 6. We check for a new path and see that path #2 exists.
#
# Without this loop, we would miss event #2. We're also guaranteed by the
# loader contract that no more events will be written to path #1 after
# events start being written to path #2, so we don't have to worry about
# that.
for event in self._loader.Load():
yield event
logger.info(
"Directory watcher advancing from %s to %s",
self._path,
next_path,
)
# Advance to the next path and start over.
self._SetPath(next_path)
# The number of paths before the current one to check for out of order writes.
_OOO_WRITE_CHECK_COUNT = 20
def OutOfOrderWritesDetected(self):
"""Returns whether any out-of-order writes have been detected.
Out-of-order writes are only checked as part of the Load() iterator. Once an
out-of-order write is detected, this function will always return true.
Note that out-of-order write detection is not performed on GCS paths, so
this function will always return false.
Returns:
Whether any out-of-order write has ever been detected by this watcher.
"""
return self._ooo_writes_detected
def _InitializeLoader(self):
path = self._GetNextPath()
if path:
self._SetPath(path)
def _SetPath(self, path):
"""Sets the current path to watch for new events.
This also records the size of the old path, if any. If the size can't be
found, an error is logged.
Args:
path: The full path of the file to watch.
"""
old_path = self._path
if old_path and not io_util.IsCloudPath(old_path):
try:
# We're done with the path, so store its size.
size = tf.io.gfile.stat(old_path).length
logger.debug("Setting latest size of %s to %d", old_path, size)
self._finalized_sizes[old_path] = size
except tf.errors.OpError as e:
logger.error("Unable to get size of %s: %s", old_path, e)
self._path = path
self._loader = self._loader_factory(path)
def _GetNextPath(self):
"""Gets the next path to load from.
This function also does the checking for out-of-order writes as it iterates
through the paths.
Returns:
The next path to load events from, or None if there are no more paths.
"""
paths = sorted(
path
for path in io_wrapper.ListDirectoryAbsolute(self._directory)
if self._path_filter(path)
)
if not paths:
return None
if self._path is None:
return paths[0]
# Don't bother checking if the paths are GCS (which we can't check) or if
# we've already detected an OOO write.
if not io_util.IsCloudPath(paths[0]) and not self._ooo_writes_detected:
# Check the previous _OOO_WRITE_CHECK_COUNT paths for out of order writes.
current_path_index = bisect.bisect_left(paths, self._path)
ooo_check_start = max(
0, current_path_index - self._OOO_WRITE_CHECK_COUNT
)
for path in paths[ooo_check_start:current_path_index]:
if self._HasOOOWrite(path):
self._ooo_writes_detected = True
break
next_paths = list(
path for path in paths if self._path is None or path > self._path
)
if next_paths:
return min(next_paths)
else:
return None
def _HasOOOWrite(self, path):
"""Returns whether the path has had an out-of-order write."""
# Check the sizes of each path before the current one.
size = tf.io.gfile.stat(path).length
old_size = self._finalized_sizes.get(path, None)
if size != old_size:
if old_size is None:
logger.error(
"File %s created after file %s even though it's "
"lexicographically earlier",
path,
self._path,
)
else:
logger.error(
"File %s updated even though the current file is %s",
path,
self._path,
)
return True
else:
return False
class DirectoryDeletedError(Exception):
"""Thrown by Load() when the directory is *permanently* gone.
We distinguish this from temporary errors so that other code can
decide to drop all of our data only when a directory has been
intentionally deleted, as opposed to due to transient filesystem
errors.
"""
pass
@@ -0,0 +1,951 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Takes a generator of values, and accumulates them for a frontend."""
import collections
import dataclasses
import threading
from typing import Optional, Sequence, Tuple
from tensorboard.backend.event_processing import directory_watcher
from tensorboard.backend.event_processing import event_file_loader
from tensorboard.backend.event_processing import event_util
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.backend.event_processing import plugin_asset_util
from tensorboard.backend.event_processing import reservoir
from tensorboard.backend.event_processing import tag_types
from tensorboard.compat.proto import config_pb2
from tensorboard.compat.proto import event_pb2
from tensorboard.compat.proto import graph_pb2
from tensorboard.compat.proto import meta_graph_pb2
from tensorboard.compat.proto import tensor_pb2
from tensorboard.plugins.distribution import compressor
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
@dataclasses.dataclass(frozen=True)
class ScalarEvent:
"""Contains information of a scalar event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
value: A float or int value of the scalar.
"""
wall_time: float
step: int
value: float
@dataclasses.dataclass(frozen=True)
class CompressedHistogramEvent:
"""Contains information of a compressed histogram event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
compressed_histogram_values: A sequence of tuples of basis points and
associated values in a compressed histogram.
"""
wall_time: float
step: int
compressed_histogram_values: Sequence[Tuple[float, float]]
@dataclasses.dataclass(frozen=True)
class HistogramValue:
"""Holds the information of the histogram values.
Attributes:
min: A float or int min value.
max: A float or int max value.
num: Total number of values.
sum: Sum of all values.
sum_squares: Sum of squares for all values.
bucket_limit: Upper values per bucket.
bucket: Numbers of values per bucket.
"""
min: float
max: float
num: int
sum: float
sum_squares: float
bucket_limit: Sequence[float]
bucket: Sequence[int]
@dataclasses.dataclass(frozen=True)
class HistogramEvent:
"""Contains information of a histogram event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
histogram_value: Information of the histogram values.
"""
wall_time: float
step: int
histogram_value: HistogramValue
@dataclasses.dataclass(frozen=True)
class ImageEvent:
"""Contains information of an image event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
encoded_image_string: Image content encoded in bytes.
width: Width of the image.
height: Height of the image.
"""
wall_time: float
step: int
encoded_image_string: bytes
width: int
height: int
@dataclasses.dataclass(frozen=True)
class AudioEvent:
"""Contains information of an audio event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
encoded_audio_string: Audio content encoded in bytes.
content_type: A string describes the type of the audio content.
sample_rate: Sample rate of the audio in Hz. Must be positive.
length_frames: Length of the audio in frames (samples per channel).
"""
wall_time: float
step: int
encoded_audio_string: bytes
content_type: str
sample_rate: float
length_frames: int
@dataclasses.dataclass(frozen=True)
class TensorEvent:
"""A tensor event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
tensor_proto: A `TensorProto`.
"""
wall_time: float
step: int
tensor_proto: tensor_pb2.TensorProto
## Different types of summary events handled by the event_accumulator
SUMMARY_TYPES = {
"simple_value": "_ProcessScalar",
"histo": "_ProcessHistogram",
"image": "_ProcessImage",
"audio": "_ProcessAudio",
"tensor": "_ProcessTensor",
}
# Legacy aliases
COMPRESSED_HISTOGRAMS = tag_types.COMPRESSED_HISTOGRAMS
HISTOGRAMS = tag_types.HISTOGRAMS
IMAGES = tag_types.IMAGES
AUDIO = tag_types.AUDIO
SCALARS = tag_types.SCALARS
TENSORS = tag_types.TENSORS
GRAPH = tag_types.GRAPH
META_GRAPH = tag_types.META_GRAPH
RUN_METADATA = tag_types.RUN_METADATA
## Normal CDF for std_devs: (-Inf, -1.5, -1, -0.5, 0, 0.5, 1, 1.5, Inf)
## naturally gives bands around median of width 1 std dev, 2 std dev, 3 std dev,
## and then the long tail.
NORMAL_HISTOGRAM_BPS = (0, 668, 1587, 3085, 5000, 6915, 8413, 9332, 10000)
DEFAULT_SIZE_GUIDANCE = {
COMPRESSED_HISTOGRAMS: 500,
IMAGES: 4,
AUDIO: 4,
SCALARS: 10000,
HISTOGRAMS: 1,
TENSORS: 10,
}
STORE_EVERYTHING_SIZE_GUIDANCE = {
COMPRESSED_HISTOGRAMS: 0,
IMAGES: 0,
AUDIO: 0,
SCALARS: 0,
HISTOGRAMS: 0,
TENSORS: 0,
}
class EventAccumulator:
"""An `EventAccumulator` takes an event generator, and accumulates the
values.
The `EventAccumulator` is intended to provide a convenient Python interface
for loading Event data written during a TensorFlow run. TensorFlow writes out
`Event` protobuf objects, which have a timestamp and step number, and often
contain a `Summary`. Summaries can have different kinds of data like an image,
a scalar value, or a histogram. The Summaries also have a tag, which we use to
organize logically related data. The `EventAccumulator` supports retrieving
the `Event` and `Summary` data by its tag.
Calling `Tags()` gets a map from `tagType` (e.g. `'images'`,
`'compressedHistograms'`, `'scalars'`, etc) to the associated tags for those
data types. Then, various functional endpoints (eg
`Accumulator.Scalars(tag)`) allow for the retrieval of all data
associated with that tag.
The `Reload()` method synchronously loads all of the data written so far.
Histograms, audio, and images are very large, so storing all of them is not
recommended.
Fields:
audios: A reservoir.Reservoir of audio summaries.
compressed_histograms: A reservoir.Reservoir of compressed
histogram summaries.
histograms: A reservoir.Reservoir of histogram summaries.
images: A reservoir.Reservoir of image summaries.
most_recent_step: Step of last Event proto added. This should only
be accessed from the thread that calls Reload. This is -1 if
nothing has been loaded yet.
most_recent_wall_time: Timestamp of last Event proto added. This is
a float containing seconds from the UNIX epoch, or -1 if
nothing has been loaded yet. This should only be accessed from
the thread that calls Reload.
path: A file path to a directory containing tf events files, or a single
tf events file. The accumulator will load events from this path.
scalars: A reservoir.Reservoir of scalar summaries.
tensors: A reservoir.Reservoir of tensor summaries.
@@Tensors
"""
def __init__(
self,
path,
size_guidance=None,
compression_bps=NORMAL_HISTOGRAM_BPS,
purge_orphaned_data=True,
):
"""Construct the `EventAccumulator`.
Args:
path: A file path to a directory containing tf events files, or a single
tf events file. The accumulator will load events from this path.
size_guidance: Information on how much data the EventAccumulator should
store in memory. The DEFAULT_SIZE_GUIDANCE tries not to store too much
so as to avoid OOMing the client. The size_guidance should be a map
from a `tagType` string to an integer representing the number of
items to keep per tag for items of that `tagType`. If the size is 0,
all events are stored.
compression_bps: Information on how the `EventAccumulator` should compress
histogram data for the `CompressedHistograms` tag (for details see
`ProcessCompressedHistogram`).
purge_orphaned_data: Whether to discard any events that were "orphaned" by
a TensorFlow restart.
"""
size_guidance = size_guidance or DEFAULT_SIZE_GUIDANCE
sizes = {}
for key in DEFAULT_SIZE_GUIDANCE:
if key in size_guidance:
sizes[key] = size_guidance[key]
else:
sizes[key] = DEFAULT_SIZE_GUIDANCE[key]
self._first_event_timestamp = None
self.scalars = reservoir.Reservoir(size=sizes[SCALARS])
self._graph = None
self._graph_from_metagraph = False
self._meta_graph = None
self._tagged_metadata = {}
self.summary_metadata = {}
self.histograms = reservoir.Reservoir(size=sizes[HISTOGRAMS])
self.compressed_histograms = reservoir.Reservoir(
size=sizes[COMPRESSED_HISTOGRAMS], always_keep_last=False
)
self.images = reservoir.Reservoir(size=sizes[IMAGES])
self.audios = reservoir.Reservoir(size=sizes[AUDIO])
self.tensors = reservoir.Reservoir(size=sizes[TENSORS])
# Keep a mapping from plugin name to a dict mapping from tag to plugin data
# content obtained from the SummaryMetadata (metadata field of Value) for
# that plugin (This is not the entire SummaryMetadata proto - only the
# content for that plugin). The SummaryWriter only keeps the content on the
# first event encountered per tag, so we must store that first instance of
# content for each tag.
self._plugin_to_tag_to_content = collections.defaultdict(dict)
self._generator_mutex = threading.Lock()
self.path = path
self._generator = _GeneratorFromPath(path)
self._compression_bps = compression_bps
self.purge_orphaned_data = purge_orphaned_data
self.most_recent_step = -1
self.most_recent_wall_time = -1
self.file_version = None
# Name of the source writer that writes the event.
self._source_writer = None
# The attributes that get built up by the accumulator
self.accumulated_attrs = (
"scalars",
"histograms",
"compressed_histograms",
"images",
"audios",
)
self._tensor_summaries = {}
def Reload(self):
"""Loads all events added since the last call to `Reload`.
If `Reload` was never called, loads all events in the file.
Returns:
The `EventAccumulator`.
"""
with self._generator_mutex:
for event in self._generator.Load():
self._ProcessEvent(event)
return self
def PluginAssets(self, plugin_name):
"""Return a list of all plugin assets for the given plugin.
Args:
plugin_name: The string name of a plugin to retrieve assets for.
Returns:
A list of string plugin asset names, or empty list if none are available.
If the plugin was not registered, an empty list is returned.
"""
return plugin_asset_util.ListAssets(self.path, plugin_name)
def RetrievePluginAsset(self, plugin_name, asset_name):
"""Return the contents of a given plugin asset.
Args:
plugin_name: The string name of a plugin.
asset_name: The string name of an asset.
Returns:
The string contents of the plugin asset.
Raises:
KeyError: If the asset is not available.
"""
return plugin_asset_util.RetrieveAsset(
self.path, plugin_name, asset_name
)
def FirstEventTimestamp(self):
"""Returns the timestamp in seconds of the first event.
If the first event has been loaded (either by this method or by `Reload`,
this returns immediately. Otherwise, it will load in the first event. Note
that this means that calling `Reload` will cause this to block until
`Reload` has finished.
Returns:
The timestamp in seconds of the first event that was loaded.
Raises:
ValueError: If no events have been loaded and there were no events found
on disk.
"""
if self._first_event_timestamp is not None:
return self._first_event_timestamp
with self._generator_mutex:
try:
event = next(self._generator.Load())
self._ProcessEvent(event)
return self._first_event_timestamp
except StopIteration:
raise ValueError("No event timestamp could be found")
def GetSourceWriter(self) -> Optional[str]:
"""Returns the name of the event writer."""
if self._source_writer is not None:
return self._source_writer
with self._generator_mutex:
try:
event = next(self._generator.Load())
self._ProcessEvent(event)
return self._source_writer
except StopIteration:
logger.info(
"End of file in %s, no source writer was found.", self.path
)
def PluginTagToContent(self, plugin_name):
"""Returns a dict mapping tags to content specific to that plugin.
Args:
plugin_name: The name of the plugin for which to fetch plugin-specific
content.
Raises:
KeyError: if the plugin name is not found.
Returns:
A dict mapping tag names to bytestrings of plugin-specific content-- by
convention, in the form of binary serialized protos.
"""
if plugin_name not in self._plugin_to_tag_to_content:
raise KeyError("Plugin %r could not be found." % plugin_name)
return self._plugin_to_tag_to_content[plugin_name]
def SummaryMetadata(self, tag):
"""Given a summary tag name, return the associated metadata object.
Args:
tag: The name of a tag, as a string.
Raises:
KeyError: If the tag is not found.
Returns:
A `SummaryMetadata` protobuf.
"""
return self.summary_metadata[tag]
def _ProcessEvent(self, event):
"""Called whenever an event is loaded."""
if self._first_event_timestamp is None:
self._first_event_timestamp = event.wall_time
if event.HasField("source_metadata"):
new_source_writer = event_util.GetSourceWriter(
event.source_metadata
)
if self._source_writer and self._source_writer != new_source_writer:
logger.info(
(
"Found new source writer for event.proto. "
"Old: {0}, New: {1}"
).format(self._source_writer, new_source_writer)
)
self._source_writer = new_source_writer
if event.HasField("file_version"):
new_file_version = event_util.ParseFileVersion(event.file_version)
if self.file_version and self.file_version != new_file_version:
## This should not happen.
logger.warning(
(
"Found new file_version for event.proto. This will "
"affect purging logic for TensorFlow restarts. "
"Old: {0} New: {1}"
).format(self.file_version, new_file_version)
)
self.file_version = new_file_version
self._MaybePurgeOrphanedData(event)
## Process the event.
# GraphDef and MetaGraphDef are handled in a special way:
# If no graph_def Event is available, but a meta_graph_def is, and it
# contains a graph_def, then use the meta_graph_def.graph_def as our graph.
# If a graph_def Event is available, always prefer it to the graph_def
# inside the meta_graph_def.
if event.HasField("graph_def"):
if self._graph is not None:
logger.warning(
(
"Found more than one graph event per run, or there was "
"a metagraph containing a graph_def, as well as one or "
"more graph events. Overwriting the graph with the "
"newest event."
)
)
self._graph = event.graph_def
self._graph_from_metagraph = False
elif event.HasField("meta_graph_def"):
if self._meta_graph is not None:
logger.warning(
(
"Found more than one metagraph event per run. "
"Overwriting the metagraph with the newest event."
)
)
self._meta_graph = event.meta_graph_def
if self._graph is None or self._graph_from_metagraph:
# We may have a graph_def in the metagraph. If so, and no
# graph_def is directly available, use this one instead.
meta_graph = meta_graph_pb2.MetaGraphDef()
meta_graph.ParseFromString(self._meta_graph)
if meta_graph.graph_def:
if self._graph is not None:
logger.warning(
(
"Found multiple metagraphs containing graph_defs,"
"but did not find any graph events. Overwriting the "
"graph with the newest metagraph version."
)
)
self._graph_from_metagraph = True
self._graph = meta_graph.graph_def.SerializeToString()
elif event.HasField("tagged_run_metadata"):
tag = event.tagged_run_metadata.tag
if tag in self._tagged_metadata:
logger.warning(
'Found more than one "run metadata" event with tag '
+ tag
+ ". Overwriting it with the newest event."
)
self._tagged_metadata[tag] = event.tagged_run_metadata.run_metadata
elif event.HasField("summary"):
for value in event.summary.value:
if value.HasField("metadata"):
tag = value.tag
# We only store the first instance of the metadata. This check
# is important: the `FileWriter` does strip metadata from all
# values except the first one per each tag, but a new
# `FileWriter` is created every time a training job stops and
# restarts. Hence, we must also ignore non-initial metadata in
# this logic.
if tag not in self.summary_metadata:
self.summary_metadata[tag] = value.metadata
plugin_data = value.metadata.plugin_data
if plugin_data.plugin_name:
self._plugin_to_tag_to_content[
plugin_data.plugin_name
][tag] = plugin_data.content
else:
logger.warning(
(
"This summary with tag %r is oddly not associated with a "
"plugin."
),
tag,
)
for summary_type, summary_func in SUMMARY_TYPES.items():
if value.HasField(summary_type):
datum = getattr(value, summary_type)
tag = value.tag
if summary_type == "tensor" and not tag:
# This tensor summary was created using the old method that used
# plugin assets. We must still continue to support it.
tag = value.node_name
getattr(self, summary_func)(
tag, event.wall_time, event.step, datum
)
def Tags(self):
"""Return all tags found in the value stream.
Returns:
A `{tagType: ['list', 'of', 'tags']}` dictionary.
"""
return {
IMAGES: self.images.Keys(),
AUDIO: self.audios.Keys(),
HISTOGRAMS: self.histograms.Keys(),
SCALARS: self.scalars.Keys(),
COMPRESSED_HISTOGRAMS: self.compressed_histograms.Keys(),
TENSORS: self.tensors.Keys(),
# Use a heuristic: if the metagraph is available, but
# graph is not, then we assume the metagraph contains the graph.
GRAPH: self._graph is not None,
META_GRAPH: self._meta_graph is not None,
RUN_METADATA: list(self._tagged_metadata.keys()),
}
def Scalars(self, tag):
"""Given a summary tag, return all associated `ScalarEvent`s.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `ScalarEvent`s.
"""
return self.scalars.Items(tag)
def Graph(self):
"""Return the graph definition, if there is one.
If the graph is stored directly, return that. If no graph is stored
directly but a metagraph is stored containing a graph, return that.
Raises:
ValueError: If there is no graph for this run.
Returns:
The `graph_def` proto.
"""
graph = graph_pb2.GraphDef()
if self._graph is not None:
graph.ParseFromString(self._graph)
return graph
raise ValueError("There is no graph in this EventAccumulator")
def MetaGraph(self):
"""Return the metagraph definition, if there is one.
Raises:
ValueError: If there is no metagraph for this run.
Returns:
The `meta_graph_def` proto.
"""
if self._meta_graph is None:
raise ValueError("There is no metagraph in this EventAccumulator")
meta_graph = meta_graph_pb2.MetaGraphDef()
meta_graph.ParseFromString(self._meta_graph)
return meta_graph
def RunMetadata(self, tag):
"""Given a tag, return the associated session.run() metadata.
Args:
tag: A string tag associated with the event.
Raises:
ValueError: If the tag is not found.
Returns:
The metadata in form of `RunMetadata` proto.
"""
if tag not in self._tagged_metadata:
raise ValueError("There is no run metadata with this tag name")
run_metadata = config_pb2.RunMetadata()
run_metadata.ParseFromString(self._tagged_metadata[tag])
return run_metadata
def Histograms(self, tag):
"""Given a summary tag, return all associated histograms.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `HistogramEvent`s.
"""
return self.histograms.Items(tag)
def CompressedHistograms(self, tag):
"""Given a summary tag, return all associated compressed histograms.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `CompressedHistogramEvent`s.
"""
return self.compressed_histograms.Items(tag)
def Images(self, tag):
"""Given a summary tag, return all associated images.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `ImageEvent`s.
"""
return self.images.Items(tag)
def Audio(self, tag):
"""Given a summary tag, return all associated audio.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `AudioEvent`s.
"""
return self.audios.Items(tag)
def Tensors(self, tag):
"""Given a summary tag, return all associated tensors.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `TensorEvent`s.
"""
return self.tensors.Items(tag)
def _MaybePurgeOrphanedData(self, event):
"""Maybe purge orphaned data due to a TensorFlow crash.
When TensorFlow crashes at step T+O and restarts at step T, any events
written after step T are now "orphaned" and will be at best misleading if
they are included in TensorBoard.
This logic attempts to determine if there is orphaned data, and purge it
if it is found.
Args:
event: The event to use as a reference, to determine if a purge is needed.
"""
if not self.purge_orphaned_data:
return
## Check if the event happened after a crash, and purge expired tags.
if self.file_version and self.file_version >= 2:
## If the file_version is recent enough, use the SessionLog enum
## to check for restarts.
self._CheckForRestartAndMaybePurge(event)
else:
## If there is no file version, default to old logic of checking for
## out of order steps.
self._CheckForOutOfOrderStepAndMaybePurge(event)
def _CheckForRestartAndMaybePurge(self, event):
"""Check and discard expired events using SessionLog.START.
Check for a SessionLog.START event and purge all previously seen events
with larger steps, because they are out of date. Because of supervisor
threading, it is possible that this logic will cause the first few event
messages to be discarded since supervisor threading does not guarantee
that the START message is deterministically written first.
This method is preferred over _CheckForOutOfOrderStepAndMaybePurge which
can inadvertently discard events due to supervisor threading.
Args:
event: The event to use as reference. If the event is a START event, all
previously seen events with a greater event.step will be purged.
"""
if (
event.HasField("session_log")
and event.session_log.status == event_pb2.SessionLog.START
):
self._Purge(event, by_tags=False)
def _CheckForOutOfOrderStepAndMaybePurge(self, event):
"""Check for out-of-order event.step and discard expired events for
tags.
Check if the event is out of order relative to the global most recent step.
If it is, purge outdated summaries for tags that the event contains.
Args:
event: The event to use as reference. If the event is out-of-order, all
events with the same tags, but with a greater event.step will be purged.
"""
if event.step < self.most_recent_step and event.HasField("summary"):
self._Purge(event, by_tags=True)
else:
self.most_recent_step = event.step
self.most_recent_wall_time = event.wall_time
def _ConvertHistogramProtoToPopo(self, histo):
"""Converts histogram proto to Python object."""
return HistogramValue(
min=histo.min,
max=histo.max,
num=histo.num,
sum=histo.sum,
sum_squares=histo.sum_squares,
bucket_limit=list(histo.bucket_limit),
bucket=list(histo.bucket),
)
def _ProcessHistogram(self, tag, wall_time, step, histo):
"""Processes a proto histogram by adding it to accumulated state."""
histo = self._ConvertHistogramProtoToPopo(histo)
histo_ev = HistogramEvent(wall_time, step, histo)
self.histograms.AddItem(tag, histo_ev)
self.compressed_histograms.AddItem(
tag, histo_ev, self._CompressHistogram
)
def _CompressHistogram(self, histo_ev):
"""Callback for _ProcessHistogram."""
return CompressedHistogramEvent(
histo_ev.wall_time,
histo_ev.step,
compressor.compress_histogram_proto(
histo_ev.histogram_value, self._compression_bps
),
)
def _ProcessImage(self, tag, wall_time, step, image):
"""Processes an image by adding it to accumulated state."""
event = ImageEvent(
wall_time=wall_time,
step=step,
encoded_image_string=image.encoded_image_string,
width=image.width,
height=image.height,
)
self.images.AddItem(tag, event)
def _ProcessAudio(self, tag, wall_time, step, audio):
"""Processes a audio by adding it to accumulated state."""
event = AudioEvent(
wall_time=wall_time,
step=step,
encoded_audio_string=audio.encoded_audio_string,
content_type=audio.content_type,
sample_rate=audio.sample_rate,
length_frames=audio.length_frames,
)
self.audios.AddItem(tag, event)
def _ProcessScalar(self, tag, wall_time, step, scalar):
"""Processes a simple value by adding it to accumulated state."""
sv = ScalarEvent(wall_time=wall_time, step=step, value=scalar)
self.scalars.AddItem(tag, sv)
def _ProcessTensor(self, tag, wall_time, step, tensor):
tv = TensorEvent(wall_time=wall_time, step=step, tensor_proto=tensor)
self.tensors.AddItem(tag, tv)
def _Purge(self, event, by_tags):
"""Purge all events that have occurred after the given event.step.
If by_tags is True, purge all events that occurred after the given
event.step, but only for the tags that the event has. Non-sequential
event.steps suggest that a TensorFlow restart occurred, and we discard
the out-of-order events to display a consistent view in TensorBoard.
Discarding by tags is the safer method, when we are unsure whether a restart
has occurred, given that threading in supervisor can cause events of
different tags to arrive with unsynchronized step values.
If by_tags is False, then purge all events with event.step greater than the
given event.step. This can be used when we are certain that a TensorFlow
restart has occurred and these events can be discarded.
Args:
event: The event to use as reference for the purge. All events with
the same tags, but with a greater event.step will be purged.
by_tags: Bool to dictate whether to discard all out-of-order events or
only those that are associated with the given reference event.
"""
## Keep data in reservoirs that has a step less than event.step
_NotExpired = lambda x: x.step < event.step
if by_tags:
def _ExpiredPerTag(value):
return [
getattr(self, x).FilterItems(_NotExpired, value.tag)
for x in self.accumulated_attrs
]
expired_per_tags = [
_ExpiredPerTag(value) for value in event.summary.value
]
expired_per_type = [sum(x) for x in zip(*expired_per_tags)]
else:
expired_per_type = [
getattr(self, x).FilterItems(_NotExpired)
for x in self.accumulated_attrs
]
if sum(expired_per_type) > 0:
purge_msg = _GetPurgeMessage(
self.most_recent_step,
self.most_recent_wall_time,
event.step,
event.wall_time,
*expired_per_type,
)
logger.warning(purge_msg)
def _GetPurgeMessage(
most_recent_step,
most_recent_wall_time,
event_step,
event_wall_time,
num_expired_scalars,
num_expired_histos,
num_expired_comp_histos,
num_expired_images,
num_expired_audio,
):
"""Return the string message associated with TensorBoard purges."""
return (
"Detected out of order event.step likely caused by "
"a TensorFlow restart. Purging expired events from Tensorboard"
" display between the previous step: {} (timestamp: {}) and "
"current step: {} (timestamp: {}). Removing {} scalars, {} "
"histograms, {} compressed histograms, {} images, "
"and {} audio."
).format(
most_recent_step,
most_recent_wall_time,
event_step,
event_wall_time,
num_expired_scalars,
num_expired_histos,
num_expired_comp_histos,
num_expired_images,
num_expired_audio,
)
def _GeneratorFromPath(path):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.LegacyEventFileLoader(path)
else:
return directory_watcher.DirectoryWatcher(
path,
event_file_loader.LegacyEventFileLoader,
io_wrapper.IsSummaryEventsFile,
)
@@ -0,0 +1,465 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Logic for TensorBoard inspector to help humans investigate event files.
Example usages:
tensorboard --inspect --event_file myevents.out
tensorboard --inspect --event_file myevents.out --tag loss
tensorboard --inspect --logdir mylogdir
tensorboard --inspect --logdir mylogdir --tag loss
This script runs over a logdir and creates an InspectionUnit for every
subdirectory with event files. If running over an event file, it creates only
one InspectionUnit. One block of output is printed to console for each
InspectionUnit.
The primary content of an InspectionUnit is the dict field_to_obs that maps
fields (e.g. "scalar", "histogram", "session_log:start", etc.) to a list of
Observations for the field. Observations correspond one-to-one with Events in an
event file but contain less information because they only store what is
necessary to generate the final console output.
The final output is rendered to console by applying some aggregating function
to the lists of Observations. Different functions are applied depending on the
type of field. For instance, for "scalar" fields, the inspector shows aggregate
statistics. For other fields like "session_log:start", all observed steps are
printed in order to aid debugging.
[1] Query a logdir or an event file for its logged tags and summary statistics
using --logdir or --event_file.
[[event_file]] contains these tags:
histograms
binary/Sign/Activations
binary/nn_tanh/act/Activations
binary/nn_tanh/biases
binary/nn_tanh/biases:gradient
binary/nn_tanh/weights
binary/nn_tanh/weights:gradient
images
input_images/image/0
input_images/image/1
input_images/image/2
scalars
Learning Rate
Total Cost
Total Cost (raw)
Debug output aggregated over all tags:
graph
first_step 0
last_step 0
max_step 0
min_step 0
num_steps 1
outoforder_steps []
histograms
first_step 491
last_step 659823
max_step 659823
min_step 491
num_steps 993
outoforder_steps []
images -
scalars
first_step 0
last_step 659823
max_step 659823
min_step 0
num_steps 1985
outoforder_steps []
sessionlog:checkpoint
first_step 7129
last_step 657167
max_step 657167
min_step 7129
num_steps 99
outoforder_steps []
sessionlog:start
outoforder_steps []
steps [0L]
sessionlog:stop -
[2] Drill down into a particular tag using --tag.
Debug output for binary/Sign/Activations:
histograms
first_step 491
last_step 659823
max_step 659823
min_step 491
num_steps 993
outoforder_steps []
"""
import dataclasses
import itertools
import os
from typing import Any, Generator, Mapping
from tensorboard.backend.event_processing import event_accumulator
from tensorboard.backend.event_processing import event_file_loader
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.compat import tf
from tensorboard.compat.proto import event_pb2
# Map of field names within summary.proto to the user-facing names that this
# script outputs.
SUMMARY_TYPE_TO_FIELD = {
"simple_value": "scalars",
"histo": "histograms",
"image": "images",
"audio": "audio",
}
for summary_type in event_accumulator.SUMMARY_TYPES:
if summary_type not in SUMMARY_TYPE_TO_FIELD:
SUMMARY_TYPE_TO_FIELD[summary_type] = summary_type
# Types of summaries that we may want to query for by tag.
TAG_FIELDS = list(SUMMARY_TYPE_TO_FIELD.values())
# Summaries that we want to see every instance of.
LONG_FIELDS = ["sessionlog:start", "sessionlog:stop"]
# Summaries that we only want an abridged digest of, since they would
# take too much screen real estate otherwise.
SHORT_FIELDS = ["graph", "sessionlog:checkpoint"] + TAG_FIELDS
# All summary types that we can inspect.
TRACKED_FIELDS = SHORT_FIELDS + LONG_FIELDS
PRINT_SEPARATOR = "=" * 70 + "\n"
@dataclasses.dataclass(frozen=True)
class Observation:
"""Contains the data within each Event file that the inspector cares about.
The inspector accumulates Observations as it processes events.
Attributes:
step: Global step of the event.
wall_time: Timestamp of the event in seconds.
tag: Tag name associated with the event.
"""
step: int
wall_time: float
tag: str
@dataclasses.dataclass(frozen=True)
class InspectionUnit:
"""Created for each organizational structure in the event files.
An InspectionUnit is visible in the final terminal output. For instance, one
InspectionUnit is created for each subdirectory in logdir. When asked to inspect
a single event file, there may only be one InspectionUnit.
Attributes:
name: Name of the organizational unit that will be printed to console.
generator: A generator that yields `Event` protos.
field_to_obs: A mapping from string fields to `Observations` that the inspector
creates.
"""
name: str
generator: Generator[event_pb2.Event, Any, Any]
field_to_obs: Mapping[str, Observation]
def get_field_to_observations_map(generator, query_for_tag=""):
"""Return a field to `Observations` dict for the event generator.
Args:
generator: A generator over event protos.
query_for_tag: A string that if specified, only create observations for
events with this tag name.
Returns:
A dict mapping keys in `TRACKED_FIELDS` to an `Observation` list.
"""
def increment(stat, event, tag=""):
assert stat in TRACKED_FIELDS
field_to_obs[stat].append(
dataclasses.asdict(
Observation(step=event.step, wall_time=event.wall_time, tag=tag)
)
)
field_to_obs = dict([(t, []) for t in TRACKED_FIELDS])
for event in generator:
## Process the event
if event.HasField("graph_def") and (not query_for_tag):
increment("graph", event)
if event.HasField("session_log") and (not query_for_tag):
status = event.session_log.status
if status == event_pb2.SessionLog.START:
increment("sessionlog:start", event)
elif status == event_pb2.SessionLog.STOP:
increment("sessionlog:stop", event)
elif status == event_pb2.SessionLog.CHECKPOINT:
increment("sessionlog:checkpoint", event)
elif event.HasField("summary"):
for value in event.summary.value:
if query_for_tag and value.tag != query_for_tag:
continue
for proto_name, display_name in SUMMARY_TYPE_TO_FIELD.items():
if value.HasField(proto_name):
increment(display_name, event, value.tag)
return field_to_obs
def get_unique_tags(field_to_obs):
"""Returns a dictionary of tags that a user could query over.
Args:
field_to_obs: Dict that maps string field to `Observation` list.
Returns:
A dict that maps keys in `TAG_FIELDS` to a list of string tags present in
the event files. If the dict does not have any observations of the type,
maps to an empty list so that we can render this to console.
"""
return {
field: sorted(set([x.get("tag", "") for x in observations]))
for field, observations in field_to_obs.items()
if field in TAG_FIELDS
}
def print_dict(d, show_missing=True):
"""Prints a shallow dict to console.
Args:
d: Dict to print.
show_missing: Whether to show keys with empty values.
"""
for k, v in sorted(d.items()):
if (not v) and show_missing:
# No instances of the key, so print missing symbol.
print("{} -".format(k))
elif isinstance(v, list):
# Value is a list, so print each item of the list.
print(k)
for item in v:
print(" {}".format(item))
elif isinstance(v, dict):
# Value is a dict, so print each (key, value) pair of the dict.
print(k)
for kk, vv in sorted(v.items()):
print(" {:<20} {}".format(kk, vv))
def get_dict_to_print(field_to_obs):
"""Transform the field-to-obs mapping into a printable dictionary.
Args:
field_to_obs: Dict that maps string field to `Observation` list.
Returns:
A dict with the keys and values to print to console.
"""
def compressed_steps(steps):
return {
"num_steps": len(set(steps)),
"min_step": min(steps),
"max_step": max(steps),
"last_step": steps[-1],
"first_step": steps[0],
"outoforder_steps": get_out_of_order(steps),
}
def full_steps(steps):
return {"steps": steps, "outoforder_steps": get_out_of_order(steps)}
output = {}
for field, observations in field_to_obs.items():
if not observations:
output[field] = None
continue
steps = [x["step"] for x in observations]
if field in SHORT_FIELDS:
output[field] = compressed_steps(steps)
if field in LONG_FIELDS:
output[field] = full_steps(steps)
return output
def get_out_of_order(list_of_numbers):
"""Returns elements that break the monotonically non-decreasing trend.
This is used to find instances of global step values that are "out-of-order",
which may trigger TensorBoard event discarding logic.
Args:
list_of_numbers: A list of numbers.
Returns:
A list of tuples in which each tuple are two elements are adjacent, but the
second element is lower than the first.
"""
# TODO: Consider changing this to only check for out-of-order
# steps within a particular tag.
result = []
# pylint: disable=consider-using-enumerate
for i in range(len(list_of_numbers)):
if i == 0:
continue
if list_of_numbers[i] < list_of_numbers[i - 1]:
result.append((list_of_numbers[i - 1], list_of_numbers[i]))
return result
def generators_from_logdir(logdir):
"""Returns a list of event generators for subdirectories with event files.
The number of generators returned should equal the number of directories
within logdir that contain event files. If only logdir contains event files,
returns a list of length one.
Args:
logdir: A log directory that contains event files.
Returns:
List of event generators for each subdirectory with event files.
"""
subdirs = io_wrapper.GetLogdirSubdirectories(logdir)
generators = [
itertools.chain(
*[
generator_from_event_file(os.path.join(subdir, f))
for f in tf.io.gfile.listdir(subdir)
if io_wrapper.IsTensorFlowEventsFile(os.path.join(subdir, f))
]
)
for subdir in subdirs
]
return generators
def generator_from_event_file(event_file):
"""Returns a generator that yields events from an event file."""
return event_file_loader.LegacyEventFileLoader(event_file).Load()
def get_inspection_units(logdir="", event_file="", tag=""):
"""Returns a list of InspectionUnit objects given either logdir or
event_file.
If logdir is given, the number of InspectionUnits should equal the
number of directories or subdirectories that contain event files.
If event_file is given, the number of InspectionUnits should be 1.
Args:
logdir: A log directory that contains event files.
event_file: Or, a particular event file path.
tag: An optional tag name to query for.
Returns:
A list of InspectionUnit objects.
"""
if logdir:
subdirs = io_wrapper.GetLogdirSubdirectories(logdir)
inspection_units = []
for subdir in subdirs:
generator = itertools.chain(
*[
generator_from_event_file(os.path.join(subdir, f))
for f in tf.io.gfile.listdir(subdir)
if io_wrapper.IsTensorFlowEventsFile(
os.path.join(subdir, f)
)
]
)
inspection_units.append(
InspectionUnit(
name=subdir,
generator=generator,
field_to_obs=get_field_to_observations_map(generator, tag),
)
)
if inspection_units:
print(
"Found event files in:\n{}\n".format(
"\n".join([u.name for u in inspection_units])
)
)
elif io_wrapper.IsTensorFlowEventsFile(logdir):
print(
"It seems that {} may be an event file instead of a logdir. If this "
"is the case, use --event_file instead of --logdir to pass "
"it in.".format(logdir)
)
else:
print("No event files found within logdir {}".format(logdir))
return inspection_units
elif event_file:
generator = generator_from_event_file(event_file)
return [
InspectionUnit(
name=event_file,
generator=generator,
field_to_obs=get_field_to_observations_map(generator, tag),
)
]
return []
def inspect(logdir="", event_file="", tag=""):
"""Main function for inspector that prints out a digest of event files.
Args:
logdir: A log directory that contains event files.
event_file: Or, a particular event file path.
tag: An optional tag name to query for.
Raises:
ValueError: If neither logdir and event_file are given, or both are given.
"""
print(
PRINT_SEPARATOR
+ "Processing event files... (this can take a few minutes)\n"
+ PRINT_SEPARATOR
)
inspection_units = get_inspection_units(logdir, event_file, tag)
for unit in inspection_units:
if tag:
print("Event statistics for tag {} in {}:".format(tag, unit.name))
else:
# If the user is not inspecting a particular tag, also print the list of
# all available tags that they can query.
print("These tags are in {}:".format(unit.name))
print_dict(get_unique_tags(unit.field_to_obs))
print(PRINT_SEPARATOR)
print("Event statistics for {}:".format(unit.name))
print_dict(get_dict_to_print(unit.field_to_obs), show_missing=(not tag))
print(PRINT_SEPARATOR)
@@ -0,0 +1,293 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functionality for loading events from a record file."""
import contextlib
from tensorboard import data_compat
from tensorboard import dataclass_compat
from tensorboard.compat import tf
from tensorboard.compat.proto import event_pb2
from tensorboard.util import platform_util
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
@contextlib.contextmanager
def _nullcontext():
"""Pre-Python-3.7-compatible standin for contextlib.nullcontext."""
yield
# Might as well make this a singleton.
_NULLCONTEXT = _nullcontext()
def _silence_deprecation_warnings():
"""Context manager that best-effort silences TF deprecation warnings."""
try:
# Learn this one weird trick to make TF deprecation warnings go away.
from tensorflow.python.util import deprecation
return deprecation.silence()
except (ImportError, AttributeError):
return _NULLCONTEXT
def _make_tf_record_iterator(file_path):
"""Returns an iterator over TF records for the given tfrecord file."""
# If we don't have TF at all, use the stub implementation.
if tf.__version__ == "stub":
# TODO(#1711): Reshape stub implementation to fit tf_record_iterator API
# rather than needlessly emulating the old PyRecordReader_New API.
logger.debug("Opening a stub record reader pointing at %s", file_path)
return _PyRecordReaderIterator(
tf.pywrap_tensorflow.PyRecordReader_New, file_path
)
# If PyRecordReader exists, use it, otherwise use tf_record_iterator().
# Check old first, then new, since tf_record_iterator existed previously but
# only gained the semantics we need at the time PyRecordReader was removed.
#
# TODO(#1711): Eventually remove PyRecordReader fallback once we can drop
# support for TF 2.1 and prior, and find a non-deprecated replacement for
# tf.compat.v1.io.tf_record_iterator.
try:
from tensorflow.python import pywrap_tensorflow
py_record_reader_new = pywrap_tensorflow.PyRecordReader_New
except (ImportError, AttributeError):
py_record_reader_new = None
if py_record_reader_new:
logger.debug("Opening a PyRecordReader pointing at %s", file_path)
return _PyRecordReaderIterator(py_record_reader_new, file_path)
else:
logger.debug("Opening a tf_record_iterator pointing at %s", file_path)
# TODO(#1711): Find non-deprecated replacement for tf_record_iterator.
with _silence_deprecation_warnings():
return tf.compat.v1.io.tf_record_iterator(file_path)
class _PyRecordReaderIterator:
"""Python iterator for TF Records based on PyRecordReader."""
def __init__(self, py_record_reader_new, file_path):
"""Constructs a _PyRecordReaderIterator for the given file path.
Args:
py_record_reader_new: pywrap_tensorflow.PyRecordReader_New
file_path: file path of the tfrecord file to read
"""
with tf.compat.v1.errors.raise_exception_on_not_ok_status() as status:
self._reader = py_record_reader_new(
tf.compat.as_bytes(file_path), 0, tf.compat.as_bytes(""), status
)
if not self._reader:
raise IOError(
"Failed to open a record reader pointing to %s" % file_path
)
def __iter__(self):
return self
def __next__(self):
try:
self._reader.GetNext()
except tf.errors.OutOfRangeError as e:
raise StopIteration
return self._reader.record()
next = __next__ # for python2 compatibility
class RawEventFileLoader:
"""An iterator that yields Event protos as serialized bytestrings."""
def __init__(self, file_path, detect_file_replacement=False):
"""Constructs a RawEventFileLoader for the given file path.
Args:
file_path: the event file path to read from
detect_file_replacement: if True, when Load() is called, the loader
will make a stat() call to check the size of the file. If it sees
that the file has grown, it will reopen the file entirely (while
preserving the current offset) before attempting to read from it.
Otherwise, Load() will simply poll at EOF for new data.
"""
if file_path is None:
raise ValueError("A file path is required")
self._file_path = platform_util.readahead_file_path(file_path)
self._detect_file_replacement = detect_file_replacement
self._file_size = None
self._iterator = _make_tf_record_iterator(self._file_path)
if self._detect_file_replacement and not hasattr(
self._iterator, "reopen"
):
logger.warning(
"File replacement detection requested, but not enabled because "
"TF record iterator impl does not support reopening. This "
"functionality requires TensorFlow 2.9+"
)
self._detect_file_replacement = False
def Load(self):
"""Loads all new events from disk as raw serialized proto bytestrings.
Calling Load multiple times in a row will not 'drop' events as long as the
return value is not iterated over.
Yields:
All event proto bytestrings in the file that have not been yielded yet.
"""
logger.debug("Loading events from %s", self._file_path)
if self._detect_file_replacement:
has_increased = self.CheckForIncreasedFileSize()
# Only act on the file size information if we got a concrete result.
if has_increased is not None:
if has_increased:
logger.debug(
"Reopening %s since file size has changed",
self._file_path,
)
self._iterator.close()
self._iterator.reopen()
else:
logger.debug(
"Skipping attempt to poll %s since file size has not "
"changed (still %d)",
self._file_path,
self._file_size,
)
return
while True:
try:
yield next(self._iterator)
except StopIteration:
logger.debug("End of file in %s", self._file_path)
break
except tf.errors.DataLossError as e:
# We swallow partial read exceptions; if the record was truncated
# and a later update completes it, retrying can then resume from
# the same point in the file since the iterator holds the offset.
logger.debug("Truncated record in %s (%s)", self._file_path, e)
break
logger.debug("No more events in %s", self._file_path)
def CheckForIncreasedFileSize(self):
"""Stats the file to get its updated size, returning True if it grew.
If the stat call fails or reports a smaller size than was previously
seen, then any previously cached size is left unchanged.
Returns:
boolean or None: True if the file size increased; False if it was
the same or decreased; or None if neither case could be detected
(either because the previous size had not been recorded yet, or
because the stat call for the current size failed).
"""
previous_size = self._file_size
try:
self._file_size = tf.io.gfile.stat(self._file_path).length
except tf.errors.OpError as e:
logger.error("Failed to stat %s: %s", self._file_path, e)
return None
logger.debug(
"Stat on %s got size %d, previous size %s",
self._file_path,
self._file_size,
previous_size,
)
if previous_size is None:
return None
if self._file_size > previous_size:
return True
if self._file_size < previous_size:
logger.warning(
"File %s shrank from previous size %d to size %d",
self._file_path,
previous_size,
self._file_size,
)
# In case this was transient, preserve the previously cached size,
# to avoid reporting a spurious increase next time. If the file was
# actually truncated, we can't recover anyway, so just ignore it.
self._file_size = previous_size
return False
class LegacyEventFileLoader(RawEventFileLoader):
"""An iterator that yields parsed Event protos."""
def Load(self):
"""Loads all new events from disk.
Calling Load multiple times in a row will not 'drop' events as long as the
return value is not iterated over.
Yields:
All events in the file that have not been yielded yet.
"""
for record in super().Load():
yield event_pb2.Event.FromString(record)
class EventFileLoader(LegacyEventFileLoader):
"""An iterator that passes events through read-time compat layers.
Specifically, this includes `data_compat` and `dataclass_compat`.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Track initial metadata for each tag, for `dataclass_compat`.
# This is meant to be tracked per run, not per event file, so
# there is a potential failure case when the second event file
# in a single run has no summary metadata. This only occurs when
# all of the following hold: (a) the events were written with
# the TensorFlow 1.x (not 2.x) writer, (b) the summaries were
# created by `tensorboard.summary.v1` ops and so do not undergo
# `data_compat` transformation, and (c) the file writer was
# reopened by calling `.reopen()` on it, which creates a new
# file but does not clear the tag cache. This is considered
# sufficiently improbable that we don't take extra mitigations.
self._initial_metadata = {} # from tag name to `SummaryMetadata`
def Load(self):
for event in super().Load():
event = data_compat.migrate_event(event)
events = dataclass_compat.migrate_event(
event, self._initial_metadata
)
for event in events:
yield event
class TimestampedEventFileLoader(EventFileLoader):
"""An iterator that yields (UNIX timestamp float, Event proto) pairs."""
def Load(self):
"""Loads all new events and their wall time values from disk.
Calling Load multiple times in a row will not 'drop' events as long as the
return value is not iterated over.
Yields:
Pairs of (UNIX timestamp float, Event proto) for all events in the file
that have not been yielded yet.
"""
for event in super().Load():
yield (event.wall_time, event)
@@ -0,0 +1,523 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides an interface for working with multiple event files."""
import os
import threading
from typing import Optional
from tensorboard.backend.event_processing import directory_watcher
from tensorboard.backend.event_processing import event_accumulator
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
class EventMultiplexer:
"""An `EventMultiplexer` manages access to multiple `EventAccumulator`s.
Each `EventAccumulator` is associated with a `run`, which is a self-contained
TensorFlow execution. The `EventMultiplexer` provides methods for extracting
information about events from multiple `run`s.
Example usage for loading specific runs from files:
```python
x = EventMultiplexer({'run1': 'path/to/run1', 'run2': 'path/to/run2'})
x.Reload()
```
Example usage for loading a directory where each subdirectory is a run
```python
(eg:) /parent/directory/path/
/parent/directory/path/run1/
/parent/directory/path/run1/events.out.tfevents.1001
/parent/directory/path/run1/events.out.tfevents.1002
/parent/directory/path/run2/
/parent/directory/path/run2/events.out.tfevents.9232
/parent/directory/path/run3/
/parent/directory/path/run3/events.out.tfevents.9232
x = EventMultiplexer().AddRunsFromDirectory('/parent/directory/path')
(which is equivalent to:)
x = EventMultiplexer({'run1': '/parent/directory/path/run1', 'run2':...}
```
If you would like to watch `/parent/directory/path`, wait for it to be created
(if necessary) and then periodically pick up new runs, use
`AutoloadingMultiplexer`
@@Tensors
"""
def __init__(
self, run_path_map=None, size_guidance=None, purge_orphaned_data=True
):
"""Constructor for the `EventMultiplexer`.
Args:
run_path_map: Dict `{run: path}` which specifies the
name of a run, and the path to find the associated events. If it is
None, then the EventMultiplexer initializes without any runs.
size_guidance: A dictionary mapping from `tagType` to the number of items
to store for each tag of that type. See
`event_accumulator.EventAccumulator` for details.
purge_orphaned_data: Whether to discard any events that were "orphaned" by
a TensorFlow restart.
"""
logger.info("Event Multiplexer initializing.")
self._accumulators_mutex = threading.Lock()
self._accumulators = {}
self._paths = {}
self._reload_called = False
self._size_guidance = (
size_guidance or event_accumulator.DEFAULT_SIZE_GUIDANCE
)
self.purge_orphaned_data = purge_orphaned_data
if run_path_map is not None:
logger.info(
"Event Multplexer doing initialization load for %s",
run_path_map,
)
for run, path in run_path_map.items():
self.AddRun(path, run)
logger.info("Event Multiplexer done initializing")
def AddRun(self, path, name=None):
"""Add a run to the multiplexer.
If the name is not specified, it is the same as the path.
If a run by that name exists, and we are already watching the right path,
do nothing. If we are watching a different path, replace the event
accumulator.
If `Reload` has been called, it will `Reload` the newly created
accumulators.
Args:
path: Path to the event files (or event directory) for given run.
name: Name of the run to add. If not provided, is set to path.
Returns:
The `EventMultiplexer`.
"""
name = name or path
accumulator = None
with self._accumulators_mutex:
if name not in self._accumulators or self._paths[name] != path:
if name in self._paths and self._paths[name] != path:
# TODO(@decentralion) - Make it impossible to overwrite an old path
# with a new path (just give the new path a distinct name)
logger.warning(
"Conflict for name %s: old path %s, new path %s",
name,
self._paths[name],
path,
)
logger.info("Constructing EventAccumulator for %s", path)
accumulator = event_accumulator.EventAccumulator(
path,
size_guidance=self._size_guidance,
purge_orphaned_data=self.purge_orphaned_data,
)
self._accumulators[name] = accumulator
self._paths[name] = path
if accumulator:
if self._reload_called:
accumulator.Reload()
return self
def AddRunsFromDirectory(self, path, name=None):
"""Load runs from a directory; recursively walks subdirectories.
If path doesn't exist, no-op. This ensures that it is safe to call
`AddRunsFromDirectory` multiple times, even before the directory is made.
If path is a directory, load event files in the directory (if any exist) and
recursively call AddRunsFromDirectory on any subdirectories. This mean you
can call AddRunsFromDirectory at the root of a tree of event logs and
TensorBoard will load them all.
If the `EventMultiplexer` is already loaded this will cause
the newly created accumulators to `Reload()`.
Args:
path: A string path to a directory to load runs from.
name: Optionally, what name to apply to the runs. If name is provided
and the directory contains run subdirectories, the name of each subrun
is the concatenation of the parent name and the subdirectory name. If
name is provided and the directory contains event files, then a run
is added called "name" and with the events from the path.
Raises:
ValueError: If the path exists and isn't a directory.
Returns:
The `EventMultiplexer`.
"""
logger.info("Starting AddRunsFromDirectory: %s", path)
for subdir in io_wrapper.GetLogdirSubdirectories(path):
logger.info("Adding events from directory %s", subdir)
rpath = os.path.relpath(subdir, path)
subname = os.path.join(name, rpath) if name else rpath
self.AddRun(subdir, name=subname)
logger.info("Done with AddRunsFromDirectory: %s", path)
return self
def Reload(self):
"""Call `Reload` on every `EventAccumulator`."""
logger.info("Beginning EventMultiplexer.Reload()")
self._reload_called = True
# Build a list so we're safe even if the list of accumulators is modified
# even while we're reloading.
with self._accumulators_mutex:
items = list(self._accumulators.items())
names_to_delete = set()
for name, accumulator in items:
try:
accumulator.Reload()
except (OSError, IOError) as e:
logger.error("Unable to reload accumulator '%s': %s", name, e)
except directory_watcher.DirectoryDeletedError:
names_to_delete.add(name)
with self._accumulators_mutex:
for name in names_to_delete:
logger.warning("Deleting accumulator '%s'", name)
del self._accumulators[name]
logger.info("Finished with EventMultiplexer.Reload()")
return self
def PluginAssets(self, plugin_name):
"""Get index of runs and assets for a given plugin.
Args:
plugin_name: Name of the plugin we are checking for.
Returns:
A dictionary that maps from run_name to a list of plugin
assets for that run.
"""
with self._accumulators_mutex:
# To avoid nested locks, we construct a copy of the run-accumulator map
items = list(self._accumulators.items())
return {run: accum.PluginAssets(plugin_name) for run, accum in items}
def RetrievePluginAsset(self, run, plugin_name, asset_name):
"""Return the contents for a specific plugin asset from a run.
Args:
run: The string name of the run.
plugin_name: The string name of a plugin.
asset_name: The string name of an asset.
Returns:
The string contents of the plugin asset.
Raises:
KeyError: If the asset is not available.
"""
accumulator = self.GetAccumulator(run)
return accumulator.RetrievePluginAsset(plugin_name, asset_name)
def FirstEventTimestamp(self, run):
"""Return the timestamp of the first event of the given run.
This may perform I/O if no events have been loaded yet for the run.
Args:
run: A string name of the run for which the timestamp is retrieved.
Returns:
The wall_time of the first event of the run, which will typically be
seconds since the epoch.
Raises:
KeyError: If the run is not found.
ValueError: If the run has no events loaded and there are no events on
disk to load.
"""
accumulator = self.GetAccumulator(run)
return accumulator.FirstEventTimestamp()
def GetSourceWriter(self, run) -> Optional[str]:
"""Returns the source writer name from the first event of the given run.
Assuming each run has only one source writer.
Args:
run: A string name of the run from which the event source information
is retrieved.
Returns:
Name of the writer that wrote the events in the run.
"""
accumulator = self.GetAccumulator(run)
return accumulator.GetSourceWriter()
def Scalars(self, run, tag):
"""Retrieve the scalar events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.ScalarEvents`.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Scalars(tag)
def Graph(self, run):
"""Retrieve the graph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The `GraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Graph()
def SerializedGraph(self, run):
"""Retrieve the serialized graph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The serialized form of the `GraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.SerializedGraph()
def MetaGraph(self, run):
"""Retrieve the metagraph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The `MetaGraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.MetaGraph()
def RunMetadata(self, run, tag):
"""Get the session.run() metadata associated with a TensorFlow run and
tag.
Args:
run: A string name of a TensorFlow run.
tag: A string name of the tag associated with a particular session.run().
Raises:
KeyError: If the run is not found, or the tag is not available for the
given run.
Returns:
The metadata in the form of `RunMetadata` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.RunMetadata(tag)
def Histograms(self, run, tag):
"""Retrieve the histogram events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.HistogramEvents`.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Histograms(tag)
def CompressedHistograms(self, run, tag):
"""Retrieve the compressed histogram events associated with a run and
tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.CompressedHistogramEvents`.
"""
accumulator = self.GetAccumulator(run)
return accumulator.CompressedHistograms(tag)
def Images(self, run, tag):
"""Retrieve the image events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.ImageEvents`.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Images(tag)
def Audio(self, run, tag):
"""Retrieve the audio events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.AudioEvents`.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Audio(tag)
def Tensors(self, run, tag):
"""Retrieve the tensor events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.TensorEvent`s.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Tensors(tag)
def PluginRunToTagToContent(self, plugin_name):
"""Returns a 2-layer dictionary of the form {run: {tag: content}}.
The `content` referred above is the content field of the PluginData proto
for the specified plugin within a Summary.Value proto.
Args:
plugin_name: The name of the plugin for which to fetch content.
Returns:
A dictionary of the form {run: {tag: content}}.
"""
mapping = {}
for run in self.Runs():
try:
tag_to_content = self.GetAccumulator(run).PluginTagToContent(
plugin_name
)
except KeyError:
# This run lacks content for the plugin. Try the next run.
continue
mapping[run] = tag_to_content
return mapping
def SummaryMetadata(self, run, tag):
"""Return the summary metadata for the given tag on the given run.
Args:
run: A string name of the run for which summary metadata is to be
retrieved.
tag: A string name of the tag whose summary metadata is to be
retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
A `SummaryMetadata` protobuf.
"""
accumulator = self.GetAccumulator(run)
return accumulator.SummaryMetadata(tag)
def Runs(self):
"""Return all the run names in the `EventMultiplexer`.
Returns:
```
{runName: { images: [tag1, tag2, tag3],
scalarValues: [tagA, tagB, tagC],
histograms: [tagX, tagY, tagZ],
compressedHistograms: [tagX, tagY, tagZ],
graph: true, meta_graph: true}}
```
"""
with self._accumulators_mutex:
# To avoid nested locks, we construct a copy of the run-accumulator map
items = list(self._accumulators.items())
return {run_name: accumulator.Tags() for run_name, accumulator in items}
def RunPaths(self):
"""Returns a dict mapping run names to event file paths."""
return self._paths
def GetAccumulator(self, run):
"""Returns EventAccumulator for a given run.
Args:
run: String name of run.
Returns:
An EventAccumulator object.
Raises:
KeyError: If run does not exist.
"""
with self._accumulators_mutex:
return self._accumulators[run]
@@ -0,0 +1,68 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functionality for processing events."""
from typing import Optional
from tensorboard.compat.proto import event_pb2
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
# Maxmimum length for event writer name.
_MAX_WRITER_NAME_LEN = 128
def ParseFileVersion(file_version: str) -> float:
"""Convert the string file_version in event.proto into a float.
Args:
file_version: String file_version from event.proto
Returns:
Version number as a float.
"""
tokens = file_version.split("brain.Event:")
try:
return float(tokens[-1])
except ValueError:
## This should never happen according to the definition of file_version
## specified in event.proto.
logger.warning(
(
"Invalid event.proto file_version. Defaulting to use of "
"out-of-order event.step logic for purging expired events."
)
)
return -1
def GetSourceWriter(
source_metadata: event_pb2.SourceMetadata,
) -> Optional[str]:
"""Gets the source writer name from the source metadata proto."""
writer_name = source_metadata.writer
if not writer_name:
return None
# Checks the length of the writer name.
if len(writer_name) > _MAX_WRITER_NAME_LEN:
logger.error(
"Source writer name `%s` is too long, maximum allowed length is %d.",
writer_name,
_MAX_WRITER_NAME_LEN,
)
return None
return writer_name
@@ -0,0 +1,224 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""IO helper functions."""
import collections
import os
import re
from tensorboard.compat import tf
from tensorboard.util import io_util
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
_ESCAPE_GLOB_CHARACTERS_REGEX = re.compile("([*?[])")
def PathSeparator(path):
return "/" if io_util.IsCloudPath(path) else os.sep
def IsTensorFlowEventsFile(path):
"""Check the path name to see if it is probably a TF Events file.
Args:
path: A file path to check if it is an event file.
Raises:
ValueError: If the path is an empty string.
Returns:
If path is formatted like a TensorFlowEventsFile. Dummy files such as
those created with the '.profile-empty' suffixes and meant to hold
no `Summary` protos are treated as true TensorFlowEventsFiles. For
background, see: https://github.com/tensorflow/tensorboard/issues/2084.
"""
if not path:
raise ValueError("Path must be a nonempty string")
return "tfevents" in tf.compat.as_str_any(os.path.basename(path))
def IsSummaryEventsFile(path):
"""Check whether the path is probably a TF Events file containing Summary.
Args:
path: A file path to check if it is an event file containing `Summary`
protos.
Returns:
If path is formatted like a TensorFlowEventsFile. Dummy files such as
those created with the '.profile-empty' suffixes and meant to hold
no `Summary` protos are treated as `False`. For background, see:
https://github.com/tensorflow/tensorboard/issues/2084.
"""
return IsTensorFlowEventsFile(path) and not path.endswith(".profile-empty")
def ListDirectoryAbsolute(directory):
"""Yields all files in the given directory.
The paths are absolute.
"""
return (
os.path.join(directory, path) for path in tf.io.gfile.listdir(directory)
)
def _EscapeGlobCharacters(path):
"""Escapes the glob characters in a path.
Python 3 has a glob.escape method, but python 2 lacks it, so we manually
implement this method.
Args:
path: The absolute path to escape.
Returns:
The escaped path string.
"""
drive, path = os.path.splitdrive(path)
return "%s%s" % (drive, _ESCAPE_GLOB_CHARACTERS_REGEX.sub(r"[\1]", path))
def ListRecursivelyViaGlobbing(top):
"""Recursively lists all files within the directory.
This method does not list subdirectories (in addition to regular files), and
the file paths are all absolute. If the directory does not exist, this yields
nothing.
This method does so by glob-ing deeper and deeper directories, ie
foo/*, foo/*/*, foo/*/*/* and so on until all files are listed. All file
paths are absolute, and this method lists subdirectories too.
For certain file systems, globbing via this method may prove significantly
faster than recursively walking a directory. Specifically, TF file systems
that implement TensorFlow's FileSystem.GetMatchingPaths method could save
costly disk reads by using this method. However, for other file systems, this
method might prove slower because the file system performs a walk per call to
glob (in which case it might as well just perform 1 walk).
Args:
top: A path to a directory.
Yields:
A (dir_path, file_paths) tuple for each directory/subdirectory.
"""
current_glob_string = os.path.join(_EscapeGlobCharacters(top), "*")
level = 0
while True:
logger.info("GlobAndListFiles: Starting to glob level %d", level)
glob = tf.io.gfile.glob(current_glob_string)
logger.info(
"GlobAndListFiles: %d files glob-ed at level %d", len(glob), level
)
if not glob:
# This subdirectory level lacks files. Terminate.
return
# Map subdirectory to a list of files.
pairs = collections.defaultdict(list)
for file_path in glob:
pairs[os.path.dirname(file_path)].append(file_path)
for dir_name, file_paths in pairs.items():
yield (dir_name, tuple(file_paths))
if len(pairs) == 1:
# If at any point the glob returns files that are all in a single
# directory, replace the current globbing path with that directory as the
# literal prefix. This should improve efficiency in cases where a single
# subdir is significantly deeper than the rest of the sudirs.
current_glob_string = os.path.join(list(pairs.keys())[0], "*")
# Iterate to the next level of subdirectories.
current_glob_string = os.path.join(current_glob_string, "*")
level += 1
def ListRecursivelyViaWalking(top):
"""Walks a directory tree, yielding (dir_path, file_paths) tuples.
For each of `top` and its subdirectories, yields a tuple containing the path
to the directory and the path to each of the contained files. Note that
unlike os.Walk()/tf.io.gfile.walk()/ListRecursivelyViaGlobbing, this does not
list subdirectories. The file paths are all absolute. If the directory does
not exist, this yields nothing.
Walking may be incredibly slow on certain file systems.
Args:
top: A path to a directory.
Yields:
A (dir_path, file_paths) tuple for each directory/subdirectory.
"""
for dir_path, _, filenames in tf.io.gfile.walk(top, topdown=True):
yield (
dir_path,
(os.path.join(dir_path, filename) for filename in filenames),
)
def GetLogdirSubdirectories(path):
"""Obtains all subdirectories with events files.
The order of the subdirectories returned is unspecified. The internal logic
that determines order varies by scenario.
Args:
path: The path to a directory under which to find subdirectories.
Returns:
A tuple of absolute paths of all subdirectories each with at least 1 events
file directly within the subdirectory.
Raises:
ValueError: If the path passed to the method exists and is not a directory.
"""
if not tf.io.gfile.exists(path):
# No directory to traverse.
return ()
if not tf.io.gfile.isdir(path):
raise ValueError(
"GetLogdirSubdirectories: path exists and is not a "
"directory, %s" % path
)
if io_util.IsCloudPath(path):
# Glob-ing for files can be significantly faster than recursively
# walking through directories for some file systems.
logger.info(
"GetLogdirSubdirectories: Starting to list directories via glob-ing."
)
traversal_method = ListRecursivelyViaGlobbing
else:
# For other file systems, the glob-ing based method might be slower because
# each call to glob could involve performing a recursive walk.
logger.info(
"GetLogdirSubdirectories: Starting to list directories via walking."
)
traversal_method = ListRecursivelyViaWalking
return (
subdir
for (subdir, files) in traversal_method(path)
if any(IsTensorFlowEventsFile(f) for f in files)
)
@@ -0,0 +1,105 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Load plugin assets from disk."""
import os.path
from tensorboard.compat import tf
_PLUGINS_DIR = "plugins"
def _IsDirectory(parent, item):
"""Helper that returns if parent/item is a directory."""
return tf.io.gfile.isdir(os.path.join(parent, item))
def PluginDirectory(logdir, plugin_name):
"""Returns the plugin directory for plugin_name."""
return os.path.join(logdir, _PLUGINS_DIR, plugin_name)
def ListPlugins(logdir):
"""List all the plugins that have registered assets in logdir.
If the plugins_dir does not exist, it returns an empty list. This maintains
compatibility with old directories that have no plugins written.
Args:
logdir: A directory that was created by a TensorFlow events writer.
Returns:
a list of plugin names, as strings
"""
plugins_dir = os.path.join(logdir, _PLUGINS_DIR)
try:
entries = tf.io.gfile.listdir(plugins_dir)
except tf.errors.NotFoundError:
return []
# Strip trailing slashes, which listdir() includes for some filesystems
# for subdirectories, after using them to bypass IsDirectory().
return [
x.rstrip("/")
for x in entries
if x.endswith("/") or _IsDirectory(plugins_dir, x)
]
def ListAssets(logdir, plugin_name):
"""List all the assets that are available for given plugin in a logdir.
Args:
logdir: A directory that was created by a TensorFlow summary.FileWriter.
plugin_name: A string name of a plugin to list assets for.
Returns:
A string list of available plugin assets. If the plugin subdirectory does
not exist (either because the logdir doesn't exist, or because the plugin
didn't register) an empty list is returned.
"""
plugin_dir = PluginDirectory(logdir, plugin_name)
try:
# Strip trailing slashes, which listdir() includes for some filesystems.
return [x.rstrip("/") for x in tf.io.gfile.listdir(plugin_dir)]
except tf.errors.NotFoundError:
return []
def RetrieveAsset(logdir, plugin_name, asset_name):
"""Retrieve a particular plugin asset from a logdir.
Args:
logdir: A directory that was created by a TensorFlow summary.FileWriter.
plugin_name: The plugin we want an asset from.
asset_name: The name of the requested asset.
Returns:
string contents of the plugin asset.
Raises:
KeyError: if the asset does not exist.
"""
asset_path = os.path.join(PluginDirectory(logdir, plugin_name), asset_name)
try:
with tf.io.gfile.GFile(asset_path, "r") as f:
return f.read()
except tf.errors.NotFoundError:
raise KeyError("Asset path %s not found" % asset_path)
except tf.errors.OpError as e:
raise KeyError(
"Couldn't read asset path: %s, OpError %s" % (asset_path, e)
)
@@ -0,0 +1,722 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Takes a generator of values, and accumulates them for a frontend."""
import collections
import dataclasses
import threading
from typing import Optional
from tensorboard.backend.event_processing import directory_loader
from tensorboard.backend.event_processing import directory_watcher
from tensorboard.backend.event_processing import event_file_loader
from tensorboard.backend.event_processing import event_util
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.backend.event_processing import plugin_asset_util
from tensorboard.backend.event_processing import reservoir
from tensorboard.backend.event_processing import tag_types
from tensorboard.compat.proto import config_pb2
from tensorboard.compat.proto import event_pb2
from tensorboard.compat.proto import graph_pb2
from tensorboard.compat.proto import meta_graph_pb2
from tensorboard.compat.proto import tensor_pb2
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
# Legacy aliases
TENSORS = tag_types.TENSORS
GRAPH = tag_types.GRAPH
META_GRAPH = tag_types.META_GRAPH
RUN_METADATA = tag_types.RUN_METADATA
DEFAULT_SIZE_GUIDANCE = {
TENSORS: 500,
}
STORE_EVERYTHING_SIZE_GUIDANCE = {
TENSORS: 0,
}
_TENSOR_RESERVOIR_KEY = "." # arbitrary
@dataclasses.dataclass(frozen=True)
class TensorEvent:
"""A tensor event.
Attributes:
wall_time: Timestamp of the event in seconds.
step: Global step of the event.
tensor_proto: A `TensorProto`.
"""
wall_time: float
step: int
tensor_proto: tensor_pb2.TensorProto
class EventAccumulator:
"""An `EventAccumulator` takes an event generator, and accumulates the
values.
The `EventAccumulator` is intended to provide a convenient Python
interface for loading Event data written during a TensorFlow run.
TensorFlow writes out `Event` protobuf objects, which have a timestamp
and step number, and often contain a `Summary`. Summaries can have
different kinds of data stored as arbitrary tensors. The Summaries
also have a tag, which we use to organize logically related data. The
`EventAccumulator` supports retrieving the `Event` and `Summary` data
by its tag.
Calling `Tags()` gets a map from `tagType` (i.e., `tensors`) to the
associated tags for those data types. Then, the functional endpoint
(i.g., `Accumulator.Tensors(tag)`) allows for the retrieval of all
data associated with that tag.
The `Reload()` method synchronously loads all of the data written so far.
Fields:
most_recent_step: Step of last Event proto added. This should only
be accessed from the thread that calls Reload. This is -1 if
nothing has been loaded yet.
most_recent_wall_time: Timestamp of last Event proto added. This is
a float containing seconds from the UNIX epoch, or -1 if
nothing has been loaded yet. This should only be accessed from
the thread that calls Reload.
path: A file path to a directory containing tf events files, or a single
tf events file. The accumulator will load events from this path.
tensors_by_tag: A dictionary mapping each tag name to a
reservoir.Reservoir of tensor summaries. Each such reservoir will
only use a single key, given by `_TENSOR_RESERVOIR_KEY`.
@@Tensors
"""
def __init__(
self,
path,
size_guidance=None,
tensor_size_guidance=None,
purge_orphaned_data=True,
event_file_active_filter=None,
detect_file_replacement=None,
):
"""Construct the `EventAccumulator`.
Args:
path: A file path to a directory containing tf events files, or a single
tf events file. The accumulator will load events from this path.
size_guidance: Information on how much data the EventAccumulator should
store in memory. The DEFAULT_SIZE_GUIDANCE tries not to store too much
so as to avoid OOMing the client. The size_guidance should be a map
from a `tagType` string to an integer representing the number of
items to keep per tag for items of that `tagType`. If the size is 0,
all events are stored.
tensor_size_guidance: Like `size_guidance`, but allowing finer
granularity for tensor summaries. Should be a map from the
`plugin_name` field on the `PluginData` proto to an integer
representing the number of items to keep per tag. Plugins for
which there is no entry in this map will default to the value of
`size_guidance[event_accumulator.TENSORS]`. Defaults to `{}`.
purge_orphaned_data: Whether to discard any events that were "orphaned" by
a TensorFlow restart.
event_file_active_filter: Optional predicate for determining whether an
event file latest load timestamp should be considered active. If passed,
this will enable multifile directory loading.
detect_file_replacement: Optional boolean; if True, event file loading
will try to detect when a file has been replaced with a new version
that contains additional data, by monitoring the file size.
"""
size_guidance = dict(size_guidance or DEFAULT_SIZE_GUIDANCE)
sizes = {}
for key in DEFAULT_SIZE_GUIDANCE:
if key in size_guidance:
sizes[key] = size_guidance[key]
else:
sizes[key] = DEFAULT_SIZE_GUIDANCE[key]
self._size_guidance = size_guidance
self._tensor_size_guidance = dict(tensor_size_guidance or {})
self._first_event_timestamp = None
self._graph = None
self._graph_from_metagraph = False
self._meta_graph = None
self._tagged_metadata = {}
self.summary_metadata = {}
self.tensors_by_tag = {}
self._tensors_by_tag_lock = threading.Lock()
# Keep a mapping from plugin name to a dict mapping from tag to plugin data
# content obtained from the SummaryMetadata (metadata field of Value) for
# that plugin (This is not the entire SummaryMetadata proto - only the
# content for that plugin). The SummaryWriter only keeps the content on the
# first event encountered per tag, so we must store that first instance of
# content for each tag.
self._plugin_to_tag_to_content = collections.defaultdict(dict)
# Locks the dict `_plugin_to_tag_to_content` as well as the
# dicts `_plugin_to_tag_to_content[p]` for each `p`.
self._plugin_tag_lock = threading.Lock()
self.path = path
self._generator = _GeneratorFromPath(
path, event_file_active_filter, detect_file_replacement
)
self._generator_mutex = threading.Lock()
self.purge_orphaned_data = purge_orphaned_data
self._seen_session_start = False
self.most_recent_step = -1
self.most_recent_wall_time = -1
self.file_version = None
# Name of the source writer that writes the event.
self._source_writer = None
def Reload(self):
"""Loads all events added since the last call to `Reload`.
If `Reload` was never called, loads all events in the file.
Returns:
The `EventAccumulator`.
"""
with self._generator_mutex:
for event in self._generator.Load():
self._ProcessEvent(event)
return self
def PluginAssets(self, plugin_name):
"""Return a list of all plugin assets for the given plugin.
Args:
plugin_name: The string name of a plugin to retrieve assets for.
Returns:
A list of string plugin asset names, or empty list if none are available.
If the plugin was not registered, an empty list is returned.
"""
return plugin_asset_util.ListAssets(self.path, plugin_name)
def RetrievePluginAsset(self, plugin_name, asset_name):
"""Return the contents of a given plugin asset.
Args:
plugin_name: The string name of a plugin.
asset_name: The string name of an asset.
Returns:
The string contents of the plugin asset.
Raises:
KeyError: If the asset is not available.
"""
return plugin_asset_util.RetrieveAsset(
self.path, plugin_name, asset_name
)
def FirstEventTimestamp(self):
"""Returns the timestamp in seconds of the first event.
If the first event has been loaded (either by this method or by `Reload`,
this returns immediately. Otherwise, it will load in the first event. Note
that this means that calling `Reload` will cause this to block until
`Reload` has finished.
Returns:
The timestamp in seconds of the first event that was loaded.
Raises:
ValueError: If no events have been loaded and there were no events found
on disk.
"""
if self._first_event_timestamp is not None:
return self._first_event_timestamp
with self._generator_mutex:
try:
event = next(self._generator.Load())
self._ProcessEvent(event)
return self._first_event_timestamp
except StopIteration:
raise ValueError("No event timestamp could be found")
def GetSourceWriter(self) -> Optional[str]:
"""Returns the name of the event writer."""
if self._source_writer is not None:
return self._source_writer
with self._generator_mutex:
try:
event = next(self._generator.Load())
self._ProcessEvent(event)
return self._source_writer
except StopIteration:
logger.info(
"End of file in %s, no source writer was found.", self.path
)
def PluginTagToContent(self, plugin_name):
"""Returns a dict mapping tags to content specific to that plugin.
Args:
plugin_name: The name of the plugin for which to fetch plugin-specific
content.
Raises:
KeyError: if the plugin name is not found.
Returns:
A dict mapping tag names to bytestrings of plugin-specific content-- by
convention, in the form of binary serialized protos.
"""
with self._plugin_tag_lock:
if plugin_name not in self._plugin_to_tag_to_content:
raise KeyError("Plugin %r could not be found." % plugin_name)
# Return a snapshot to avoid concurrent mutation and iteration issues.
return dict(self._plugin_to_tag_to_content[plugin_name])
def ActivePlugins(self):
"""Return a set of plugins with summary data.
Returns:
The distinct union of `plugin_data.plugin_name` fields from
all the `SummaryMetadata` protos stored in this accumulator.
"""
with self._plugin_tag_lock:
return frozenset(self._plugin_to_tag_to_content)
def SummaryMetadata(self, tag):
"""Given a summary tag name, return the associated metadata object.
Args:
tag: The name of a tag, as a string.
Raises:
KeyError: If the tag is not found.
Returns:
A `SummaryMetadata` protobuf.
"""
return self.summary_metadata[tag]
def AllSummaryMetadata(self):
"""Return summary metadata for all tags.
Returns:
A dict `d` such that `d[tag]` is a `SummaryMetadata` proto for
the keyed tag.
"""
return dict(self.summary_metadata)
def _ProcessEvent(self, event):
"""Called whenever an event is loaded."""
if self._first_event_timestamp is None:
self._first_event_timestamp = event.wall_time
if event.HasField("source_metadata"):
new_source_writer = event_util.GetSourceWriter(
event.source_metadata
)
if self._source_writer and self._source_writer != new_source_writer:
logger.info(
(
"Found new source writer for event.proto. "
"Old: {0}, New: {1}"
).format(self._source_writer, new_source_writer)
)
self._source_writer = new_source_writer
if event.HasField("file_version"):
new_file_version = event_util.ParseFileVersion(event.file_version)
if self.file_version and self.file_version != new_file_version:
## This should not happen.
logger.warning(
(
"Found new file_version for event.proto. This will "
"affect purging logic for TensorFlow restarts. "
"Old: {0} New: {1}"
).format(self.file_version, new_file_version)
)
self.file_version = new_file_version
self._MaybePurgeOrphanedData(event)
## Process the event.
# GraphDef and MetaGraphDef are handled in a special way:
# If no graph_def Event is available, but a meta_graph_def is, and it
# contains a graph_def, then use the meta_graph_def.graph_def as our graph.
# If a graph_def Event is available, always prefer it to the graph_def
# inside the meta_graph_def.
if event.HasField("graph_def"):
if self._graph is not None:
logger.warning(
(
"Found more than one graph event per run, or there was "
"a metagraph containing a graph_def, as well as one or "
"more graph events. Overwriting the graph with the "
"newest event."
)
)
self._graph = event.graph_def
self._graph_from_metagraph = False
elif event.HasField("meta_graph_def"):
if self._meta_graph is not None:
logger.warning(
(
"Found more than one metagraph event per run. "
"Overwriting the metagraph with the newest event."
)
)
self._meta_graph = event.meta_graph_def
if self._graph is None or self._graph_from_metagraph:
# We may have a graph_def in the metagraph. If so, and no
# graph_def is directly available, use this one instead.
meta_graph = meta_graph_pb2.MetaGraphDef()
meta_graph.ParseFromString(self._meta_graph)
if meta_graph.graph_def:
if self._graph is not None:
logger.warning(
(
"Found multiple metagraphs containing graph_defs,"
"but did not find any graph events. Overwriting the "
"graph with the newest metagraph version."
)
)
self._graph_from_metagraph = True
self._graph = meta_graph.graph_def.SerializeToString()
elif event.HasField("tagged_run_metadata"):
tag = event.tagged_run_metadata.tag
if tag in self._tagged_metadata:
logger.warning(
'Found more than one "run metadata" event with tag '
+ tag
+ ". Overwriting it with the newest event."
)
self._tagged_metadata[tag] = event.tagged_run_metadata.run_metadata
elif event.HasField("summary"):
for value in event.summary.value:
if value.HasField("metadata"):
tag = value.tag
# We only store the first instance of the metadata. This check
# is important: the `FileWriter` does strip metadata from all
# values except the first one per each tag, but a new
# `FileWriter` is created every time a training job stops and
# restarts. Hence, we must also ignore non-initial metadata in
# this logic.
if tag not in self.summary_metadata:
self.summary_metadata[tag] = value.metadata
plugin_data = value.metadata.plugin_data
if plugin_data.plugin_name:
with self._plugin_tag_lock:
self._plugin_to_tag_to_content[
plugin_data.plugin_name
][tag] = plugin_data.content
else:
logger.warning(
(
"This summary with tag %r is oddly not associated with a "
"plugin."
),
tag,
)
if value.HasField("tensor"):
datum = value.tensor
tag = value.tag
if not tag:
# This tensor summary was created using the old method that used
# plugin assets. We must still continue to support it.
tag = value.node_name
self._ProcessTensor(tag, event.wall_time, event.step, datum)
def Tags(self):
"""Return all tags found in the value stream.
Returns:
A `{tagType: ['list', 'of', 'tags']}` dictionary.
"""
return {
TENSORS: list(self.tensors_by_tag.keys()),
# Use a heuristic: if the metagraph is available, but
# graph is not, then we assume the metagraph contains the graph.
GRAPH: self._graph is not None,
META_GRAPH: self._meta_graph is not None,
RUN_METADATA: list(self._tagged_metadata.keys()),
}
def Graph(self):
"""Return the graph definition, if there is one.
If the graph is stored directly, return that. If no graph is stored
directly but a metagraph is stored containing a graph, return that.
Raises:
ValueError: If there is no graph for this run.
Returns:
The `graph_def` proto.
"""
graph = graph_pb2.GraphDef()
if self._graph is not None:
graph.ParseFromString(self._graph)
return graph
raise ValueError("There is no graph in this EventAccumulator")
def SerializedGraph(self):
"""Return the graph definition in serialized form, if there is one."""
return self._graph
def MetaGraph(self):
"""Return the metagraph definition, if there is one.
Raises:
ValueError: If there is no metagraph for this run.
Returns:
The `meta_graph_def` proto.
"""
if self._meta_graph is None:
raise ValueError("There is no metagraph in this EventAccumulator")
meta_graph = meta_graph_pb2.MetaGraphDef()
meta_graph.ParseFromString(self._meta_graph)
return meta_graph
def RunMetadata(self, tag):
"""Given a tag, return the associated session.run() metadata.
Args:
tag: A string tag associated with the event.
Raises:
ValueError: If the tag is not found.
Returns:
The metadata in form of `RunMetadata` proto.
"""
if tag not in self._tagged_metadata:
raise ValueError("There is no run metadata with this tag name")
run_metadata = config_pb2.RunMetadata()
run_metadata.ParseFromString(self._tagged_metadata[tag])
return run_metadata
def Tensors(self, tag):
"""Given a summary tag, return all associated tensors.
Args:
tag: A string tag associated with the events.
Raises:
KeyError: If the tag is not found.
Returns:
An array of `TensorEvent`s.
"""
return self.tensors_by_tag[tag].Items(_TENSOR_RESERVOIR_KEY)
def _MaybePurgeOrphanedData(self, event):
"""Maybe purge orphaned data due to a TensorFlow crash.
When TensorFlow crashes at step T+O and restarts at step T, any events
written after step T are now "orphaned" and will be at best misleading if
they are included in TensorBoard.
This logic attempts to determine if there is orphaned data, and purge it
if it is found.
Args:
event: The event to use as a reference, to determine if a purge is needed.
"""
if not self.purge_orphaned_data:
return
## Check if the event happened after a crash, and purge expired tags.
if self.file_version and self.file_version >= 2:
## If the file_version is recent enough, use the SessionLog enum
## to check for restarts.
self._CheckForRestartAndMaybePurge(event)
else:
## If there is no file version, default to old logic of checking for
## out of order steps.
self._CheckForOutOfOrderStepAndMaybePurge(event)
# After checking, update the most recent summary step and wall time.
if event.HasField("summary"):
self.most_recent_step = event.step
self.most_recent_wall_time = event.wall_time
def _CheckForRestartAndMaybePurge(self, event):
"""Check and discard expired events using SessionLog.START.
The first SessionLog.START event in a run indicates the start of a
supervisor session. Subsequent SessionLog.START events indicate a
*restart*, which may need to preempt old events. This method checks
for a session restart event and purges all previously seen events whose
step is larger than or equal to this event's step.
Because of supervisor threading, it is possible that this logic will
cause the first few event messages to be discarded since supervisor
threading does not guarantee that the START message is deterministically
written first.
This method is preferred over _CheckForOutOfOrderStepAndMaybePurge which
can inadvertently discard events due to supervisor threading.
Args:
event: The event to use as reference. If the event is a START event, all
previously seen events with a greater event.step will be purged.
"""
if event.session_log.status != event_pb2.SessionLog.START:
return
if not self._seen_session_start:
# Initial start event: does not indicate a restart.
self._seen_session_start = True
return
self._Purge(event, by_tags=False)
def _CheckForOutOfOrderStepAndMaybePurge(self, event):
"""Check for out-of-order event.step and discard expired events for
tags.
Check if the event is out of order relative to the global most recent step.
If it is, purge outdated summaries for tags that the event contains.
Args:
event: The event to use as reference. If the event is out-of-order, all
events with the same tags, but with a greater event.step will be purged.
"""
if event.step < self.most_recent_step and event.HasField("summary"):
self._Purge(event, by_tags=True)
def _ProcessTensor(self, tag, wall_time, step, tensor):
tv = TensorEvent(wall_time=wall_time, step=step, tensor_proto=tensor)
with self._tensors_by_tag_lock:
if tag not in self.tensors_by_tag:
reservoir_size = self._GetTensorReservoirSize(tag)
self.tensors_by_tag[tag] = reservoir.Reservoir(reservoir_size)
self.tensors_by_tag[tag].AddItem(_TENSOR_RESERVOIR_KEY, tv)
def _GetTensorReservoirSize(self, tag):
default = self._size_guidance[TENSORS]
summary_metadata = self.summary_metadata.get(tag)
if summary_metadata is None:
return default
return self._tensor_size_guidance.get(
summary_metadata.plugin_data.plugin_name, default
)
def _Purge(self, event, by_tags):
"""Purge all events that have occurred after the given event.step.
If by_tags is True, purge all events that occurred after the given
event.step, but only for the tags that the event has. Non-sequential
event.steps suggest that a TensorFlow restart occurred, and we discard
the out-of-order events to display a consistent view in TensorBoard.
Discarding by tags is the safer method, when we are unsure whether a restart
has occurred, given that threading in supervisor can cause events of
different tags to arrive with unsynchronized step values.
If by_tags is False, then purge all events with event.step greater than the
given event.step. This can be used when we are certain that a TensorFlow
restart has occurred and these events can be discarded.
Args:
event: The event to use as reference for the purge. All events with
the same tags, but with a greater event.step will be purged.
by_tags: Bool to dictate whether to discard all out-of-order events or
only those that are associated with the given reference event.
"""
## Keep data in reservoirs that has a step less than event.step
_NotExpired = lambda x: x.step < event.step
num_expired = 0
if by_tags:
for value in event.summary.value:
if value.tag in self.tensors_by_tag:
tag_reservoir = self.tensors_by_tag[value.tag]
num_expired += tag_reservoir.FilterItems(
_NotExpired, _TENSOR_RESERVOIR_KEY
)
else:
for tag_reservoir in self.tensors_by_tag.values():
num_expired += tag_reservoir.FilterItems(
_NotExpired, _TENSOR_RESERVOIR_KEY
)
if num_expired > 0:
purge_msg = _GetPurgeMessage(
self.most_recent_step,
self.most_recent_wall_time,
event.step,
event.wall_time,
num_expired,
)
logger.warning(purge_msg)
def _GetPurgeMessage(
most_recent_step,
most_recent_wall_time,
event_step,
event_wall_time,
num_expired,
):
"""Return the string message associated with TensorBoard purges."""
return (
"Detected out of order event.step likely caused by a TensorFlow "
"restart. Purging {} expired tensor events from Tensorboard display "
"between the previous step: {} (timestamp: {}) and current step: {} "
"(timestamp: {})."
).format(
num_expired,
most_recent_step,
most_recent_wall_time,
event_step,
event_wall_time,
)
def _GeneratorFromPath(
path, event_file_active_filter=None, detect_file_replacement=None
):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path, detect_file_replacement)
elif event_file_active_filter:
loader_factory = (
lambda path: event_file_loader.TimestampedEventFileLoader(
path, detect_file_replacement
)
)
return directory_loader.DirectoryLoader(
path,
loader_factory,
path_filter=io_wrapper.IsSummaryEventsFile,
active_filter=event_file_active_filter,
)
else:
loader_factory = lambda path: event_file_loader.EventFileLoader(
path, detect_file_replacement
)
return directory_watcher.DirectoryWatcher(
path,
loader_factory,
io_wrapper.IsSummaryEventsFile,
)
@@ -0,0 +1,524 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides an interface for working with multiple event files."""
import os
import queue
import threading
from typing import Optional
from tensorboard.backend.event_processing import directory_watcher
from tensorboard.backend.event_processing import (
plugin_event_accumulator as event_accumulator,
)
from tensorboard.backend.event_processing import io_wrapper
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
class EventMultiplexer:
"""An `EventMultiplexer` manages access to multiple `EventAccumulator`s.
Each `EventAccumulator` is associated with a `run`, which is a self-contained
TensorFlow execution. The `EventMultiplexer` provides methods for extracting
information about events from multiple `run`s.
Example usage for loading specific runs from files:
```python
x = EventMultiplexer({'run1': 'path/to/run1', 'run2': 'path/to/run2'})
x.Reload()
```
Example usage for loading a directory where each subdirectory is a run
```python
(eg:) /parent/directory/path/
/parent/directory/path/run1/
/parent/directory/path/run1/events.out.tfevents.1001
/parent/directory/path/run1/events.out.tfevents.1002
/parent/directory/path/run2/
/parent/directory/path/run2/events.out.tfevents.9232
/parent/directory/path/run3/
/parent/directory/path/run3/events.out.tfevents.9232
x = EventMultiplexer().AddRunsFromDirectory('/parent/directory/path')
(which is equivalent to:)
x = EventMultiplexer({'run1': '/parent/directory/path/run1', 'run2':...}
```
If you would like to watch `/parent/directory/path`, wait for it to be created
(if necessary) and then periodically pick up new runs, use
`AutoloadingMultiplexer`
@@Tensors
"""
def __init__(
self,
run_path_map=None,
size_guidance=None,
tensor_size_guidance=None,
purge_orphaned_data=True,
max_reload_threads=None,
event_file_active_filter=None,
detect_file_replacement=None,
):
"""Constructor for the `EventMultiplexer`.
Args:
run_path_map: Dict `{run: path}` which specifies the
name of a run, and the path to find the associated events. If it is
None, then the EventMultiplexer initializes without any runs.
size_guidance: A dictionary mapping from `tagType` to the number of items
to store for each tag of that type. See
`event_accumulator.EventAccumulator` for details.
tensor_size_guidance: A dictionary mapping from `plugin_name` to
the number of items to store for each tag of that type. See
`event_accumulator.EventAccumulator` for details.
purge_orphaned_data: Whether to discard any events that were "orphaned" by
a TensorFlow restart.
max_reload_threads: The max number of threads that TensorBoard can use
to reload runs. Each thread reloads one run at a time. If not provided,
reloads runs serially (one after another).
event_file_active_filter: Optional predicate for determining whether an
event file latest load timestamp should be considered active. If passed,
this will enable multifile directory loading.
detect_file_replacement: Optional boolean; if True, event file loading
will try to detect when a file has been replaced with a new version
that contains additional data, by monitoring the file size.
"""
logger.info("Event Multiplexer initializing.")
self._accumulators_mutex = threading.Lock()
self._accumulators = {}
self._paths = {}
self._reload_called = False
self._size_guidance = (
size_guidance or event_accumulator.DEFAULT_SIZE_GUIDANCE
)
self._tensor_size_guidance = tensor_size_guidance
self.purge_orphaned_data = purge_orphaned_data
self._max_reload_threads = max_reload_threads or 1
self._event_file_active_filter = event_file_active_filter
self._detect_file_replacement = detect_file_replacement
if run_path_map is not None:
logger.info(
"Event Multplexer doing initialization load for %s",
run_path_map,
)
for run, path in run_path_map.items():
self.AddRun(path, run)
logger.info("Event Multiplexer done initializing")
def AddRun(self, path, name=None):
"""Add a run to the multiplexer.
If the name is not specified, it is the same as the path.
If a run by that name exists, and we are already watching the right path,
do nothing. If we are watching a different path, replace the event
accumulator.
If `Reload` has been called, it will `Reload` the newly created
accumulators.
Args:
path: Path to the event files (or event directory) for given run.
name: Name of the run to add. If not provided, is set to path.
Returns:
The `EventMultiplexer`.
"""
name = name or path
accumulator = None
with self._accumulators_mutex:
if name not in self._accumulators or self._paths[name] != path:
if name in self._paths and self._paths[name] != path:
# TODO(@decentralion) - Make it impossible to overwrite an old path
# with a new path (just give the new path a distinct name)
logger.warning(
"Conflict for name %s: old path %s, new path %s",
name,
self._paths[name],
path,
)
logger.info("Constructing EventAccumulator for %s", path)
accumulator = event_accumulator.EventAccumulator(
path,
size_guidance=self._size_guidance,
tensor_size_guidance=self._tensor_size_guidance,
purge_orphaned_data=self.purge_orphaned_data,
event_file_active_filter=self._event_file_active_filter,
detect_file_replacement=self._detect_file_replacement,
)
self._accumulators[name] = accumulator
self._paths[name] = path
if accumulator:
if self._reload_called:
accumulator.Reload()
return self
def AddRunsFromDirectory(self, path, name=None):
"""Load runs from a directory; recursively walks subdirectories.
If path doesn't exist, no-op. This ensures that it is safe to call
`AddRunsFromDirectory` multiple times, even before the directory is made.
If path is a directory, load event files in the directory (if any exist) and
recursively call AddRunsFromDirectory on any subdirectories. This mean you
can call AddRunsFromDirectory at the root of a tree of event logs and
TensorBoard will load them all.
If the `EventMultiplexer` is already loaded this will cause
the newly created accumulators to `Reload()`.
Args:
path: A string path to a directory to load runs from.
name: Optionally, what name to apply to the runs. If name is provided
and the directory contains run subdirectories, the name of each subrun
is the concatenation of the parent name and the subdirectory name. If
name is provided and the directory contains event files, then a run
is added called "name" and with the events from the path.
Raises:
ValueError: If the path exists and isn't a directory.
Returns:
The `EventMultiplexer`.
"""
path = os.path.expanduser(path)
logger.info("Starting AddRunsFromDirectory: %s", path)
for subdir in io_wrapper.GetLogdirSubdirectories(path):
logger.info("Adding run from directory %s", subdir)
rpath = os.path.relpath(subdir, path)
subname = os.path.join(name, rpath) if name else rpath
self.AddRun(subdir, name=subname)
logger.info("Done with AddRunsFromDirectory: %s", path)
return self
def Reload(self):
"""Call `Reload` on every `EventAccumulator`."""
logger.info("Beginning EventMultiplexer.Reload()")
self._reload_called = True
# Build a list so we're safe even if the list of accumulators is modified
# even while we're reloading.
with self._accumulators_mutex:
items = list(self._accumulators.items())
items_queue = queue.Queue()
for item in items:
items_queue.put(item)
# Methods of built-in python containers are thread-safe so long as the GIL
# for the thread exists, but we might as well be careful.
names_to_delete = set()
names_to_delete_mutex = threading.Lock()
def Worker():
"""Keeps reloading accumulators til none are left."""
while True:
try:
name, accumulator = items_queue.get(block=False)
except queue.Empty:
# No more runs to reload.
break
try:
accumulator.Reload()
except (OSError, IOError) as e:
logger.error("Unable to reload accumulator %r: %s", name, e)
except directory_watcher.DirectoryDeletedError:
with names_to_delete_mutex:
names_to_delete.add(name)
finally:
items_queue.task_done()
if self._max_reload_threads > 1:
num_threads = min(self._max_reload_threads, len(items))
logger.info("Starting %d threads to reload runs", num_threads)
for i in range(num_threads):
thread = threading.Thread(target=Worker, name="Reloader %d" % i)
thread.daemon = True
thread.start()
items_queue.join()
else:
logger.info(
"Reloading runs serially (one after another) on the main "
"thread."
)
Worker()
with self._accumulators_mutex:
for name in names_to_delete:
logger.warning("Deleting accumulator %r", name)
del self._accumulators[name]
logger.info("Finished with EventMultiplexer.Reload()")
return self
def PluginAssets(self, plugin_name):
"""Get index of runs and assets for a given plugin.
Args:
plugin_name: Name of the plugin we are checking for.
Returns:
A dictionary that maps from run_name to a list of plugin
assets for that run.
"""
with self._accumulators_mutex:
# To avoid nested locks, we construct a copy of the run-accumulator map
items = list(self._accumulators.items())
return {run: accum.PluginAssets(plugin_name) for run, accum in items}
def RetrievePluginAsset(self, run, plugin_name, asset_name):
"""Return the contents for a specific plugin asset from a run.
Args:
run: The string name of the run.
plugin_name: The string name of a plugin.
asset_name: The string name of an asset.
Returns:
The string contents of the plugin asset.
Raises:
KeyError: If the asset is not available.
"""
accumulator = self.GetAccumulator(run)
return accumulator.RetrievePluginAsset(plugin_name, asset_name)
def FirstEventTimestamp(self, run):
"""Return the timestamp of the first event of the given run.
This may perform I/O if no events have been loaded yet for the run.
Args:
run: A string name of the run for which the timestamp is retrieved.
Returns:
The wall_time of the first event of the run, which will typically be
seconds since the epoch.
Raises:
KeyError: If the run is not found.
ValueError: If the run has no events loaded and there are no events on
disk to load.
"""
accumulator = self.GetAccumulator(run)
return accumulator.FirstEventTimestamp()
def GetSourceWriter(self, run) -> Optional[str]:
"""Returns the source writer name from the first event of the given run.
Assuming each run has only one source writer.
Args:
run: A string name of the run from which the event source information
is retrieved.
Returns:
Name of the writer that wrote the events in the run.
"""
accumulator = self.GetAccumulator(run)
return accumulator.GetSourceWriter()
def Graph(self, run):
"""Retrieve the graph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The `GraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Graph()
def SerializedGraph(self, run):
"""Retrieve the serialized graph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The serialized form of the `GraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.SerializedGraph()
def MetaGraph(self, run):
"""Retrieve the metagraph associated with the provided run.
Args:
run: A string name of a run to load the graph for.
Raises:
KeyError: If the run is not found.
ValueError: If the run does not have an associated graph.
Returns:
The `MetaGraphDef` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.MetaGraph()
def RunMetadata(self, run, tag):
"""Get the session.run() metadata associated with a TensorFlow run and
tag.
Args:
run: A string name of a TensorFlow run.
tag: A string name of the tag associated with a particular session.run().
Raises:
KeyError: If the run is not found, or the tag is not available for the
given run.
Returns:
The metadata in the form of `RunMetadata` protobuf data structure.
"""
accumulator = self.GetAccumulator(run)
return accumulator.RunMetadata(tag)
def Tensors(self, run, tag):
"""Retrieve the tensor events associated with a run and tag.
Args:
run: A string name of the run for which values are retrieved.
tag: A string name of the tag for which values are retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
An array of `event_accumulator.TensorEvent`s.
"""
accumulator = self.GetAccumulator(run)
return accumulator.Tensors(tag)
def PluginRunToTagToContent(self, plugin_name):
"""Returns a 2-layer dictionary of the form {run: {tag: content}}.
The `content` referred above is the content field of the PluginData proto
for the specified plugin within a Summary.Value proto.
Args:
plugin_name: The name of the plugin for which to fetch content.
Returns:
A dictionary of the form {run: {tag: content}}.
"""
mapping = {}
for run in self.Runs():
try:
tag_to_content = self.GetAccumulator(run).PluginTagToContent(
plugin_name
)
except KeyError:
# This run lacks content for the plugin. Try the next run.
continue
mapping[run] = tag_to_content
return mapping
def ActivePlugins(self):
"""Return a set of plugins with summary data.
Returns:
The distinct union of `plugin_data.plugin_name` fields from
all the `SummaryMetadata` protos stored in any run known to
this multiplexer.
"""
with self._accumulators_mutex:
accumulators = list(self._accumulators.values())
return frozenset().union(*(a.ActivePlugins() for a in accumulators))
def SummaryMetadata(self, run, tag):
"""Return the summary metadata for the given tag on the given run.
Args:
run: A string name of the run for which summary metadata is to be
retrieved.
tag: A string name of the tag whose summary metadata is to be
retrieved.
Raises:
KeyError: If the run is not found, or the tag is not available for
the given run.
Returns:
A `SummaryMetadata` protobuf.
"""
accumulator = self.GetAccumulator(run)
return accumulator.SummaryMetadata(tag)
def AllSummaryMetadata(self):
"""Return summary metadata for all time series.
Returns:
A nested dict `d` such that `d[run][tag]` is a
`SummaryMetadata` proto for the keyed time series.
"""
with self._accumulators_mutex:
# To avoid nested locks, we construct a copy of the run-accumulator map
items = list(self._accumulators.items())
return {
run_name: accumulator.AllSummaryMetadata()
for run_name, accumulator in items
}
def Runs(self):
"""Return all the run names in the `EventMultiplexer`.
Returns:
```
{runName: { scalarValues: [tagA, tagB, tagC],
graph: true, meta_graph: true}}
```
"""
with self._accumulators_mutex:
# To avoid nested locks, we construct a copy of the run-accumulator map
items = list(self._accumulators.items())
return {run_name: accumulator.Tags() for run_name, accumulator in items}
def RunPaths(self):
"""Returns a dict mapping run names to event file paths."""
return self._paths
def GetAccumulator(self, run):
"""Returns EventAccumulator for a given run.
Args:
run: String name of run.
Returns:
An EventAccumulator object.
Raises:
KeyError: If run does not exist.
"""
with self._accumulators_mutex:
return self._accumulators[run]
@@ -0,0 +1,267 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A key-value[] store that implements reservoir sampling on the values."""
import collections
import random
import threading
class Reservoir:
"""A map-to-arrays container, with deterministic Reservoir Sampling.
Items are added with an associated key. Items may be retrieved by key, and
a list of keys can also be retrieved. If size is not zero, then it dictates
the maximum number of items that will be stored with each key. Once there are
more items for a given key, they are replaced via reservoir sampling, such
that each item has an equal probability of being included in the sample.
Deterministic means that for any given seed and bucket size, the sequence of
values that are kept for any given tag will always be the same, and that this
is independent of any insertions on other tags. That is:
>>> separate_reservoir = reservoir.Reservoir(10)
>>> interleaved_reservoir = reservoir.Reservoir(10)
>>> for i in range(100):
>>> separate_reservoir.AddItem('key1', i)
>>> for i in range(100):
>>> separate_reservoir.AddItem('key2', i)
>>> for i in range(100):
>>> interleaved_reservoir.AddItem('key1', i)
>>> interleaved_reservoir.AddItem('key2', i)
separate_reservoir and interleaved_reservoir will be in identical states.
See: https://en.wikipedia.org/wiki/Reservoir_sampling
Adding items has amortized O(1) runtime.
Fields:
always_keep_last: Whether the latest seen sample is always at the
end of the reservoir. Defaults to True.
size: An integer of the maximum number of samples.
"""
def __init__(self, size, seed=0, always_keep_last=True):
"""Creates a new reservoir.
Args:
size: The number of values to keep in the reservoir for each tag. If 0,
all values will be kept.
seed: The seed of the random number generator to use when sampling.
Different values for |seed| will produce different samples from the same
input items.
always_keep_last: Whether to always keep the latest seen item in the
end of the reservoir. Defaults to True.
Raises:
ValueError: If size is negative or not an integer.
"""
if size < 0 or size != round(size):
raise ValueError("size must be nonnegative integer, was %s" % size)
self._buckets = collections.defaultdict(
lambda: _ReservoirBucket(
size, random.Random(seed), always_keep_last
)
)
# _mutex guards the keys - creating new keys, retrieving by key, etc
# the internal items are guarded by the ReservoirBuckets' internal mutexes
self._mutex = threading.Lock()
self.size = size
self.always_keep_last = always_keep_last
def Keys(self):
"""Return all the keys in the reservoir.
Returns:
['list', 'of', 'keys'] in the Reservoir.
"""
with self._mutex:
return list(self._buckets.keys())
def Items(self, key):
"""Return items associated with given key.
Args:
key: The key for which we are finding associated items.
Raises:
KeyError: If the key is not found in the reservoir.
Returns:
[list, of, items] associated with that key.
"""
with self._mutex:
if key not in self._buckets:
raise KeyError("Key %s was not found in Reservoir" % key)
bucket = self._buckets[key]
return bucket.Items()
def AddItem(self, key, item, f=lambda x: x):
"""Add a new item to the Reservoir with the given tag.
If the reservoir has not yet reached full size, the new item is guaranteed
to be added. If the reservoir is full, then behavior depends on the
always_keep_last boolean.
If always_keep_last was set to true, the new item is guaranteed to be added
to the reservoir, and either the previous last item will be replaced, or
(with low probability) an older item will be replaced.
If always_keep_last was set to false, then the new item will replace an
old item with low probability.
If f is provided, it will be applied to transform item (lazily, iff item is
going to be included in the reservoir).
Args:
key: The key to store the item under.
item: The item to add to the reservoir.
f: An optional function to transform the item prior to addition.
"""
with self._mutex:
bucket = self._buckets[key]
bucket.AddItem(item, f)
def FilterItems(self, filterFn, key=None):
"""Filter items within a Reservoir, using a filtering function.
Args:
filterFn: A function that returns True for the items to be kept.
key: An optional bucket key to filter. If not specified, will filter all
all buckets.
Returns:
The number of items removed.
"""
with self._mutex:
if key:
if key in self._buckets:
return self._buckets[key].FilterItems(filterFn)
else:
return 0
else:
return sum(
bucket.FilterItems(filterFn)
for bucket in self._buckets.values()
)
class _ReservoirBucket:
"""A container for items from a stream, that implements reservoir sampling.
It always stores the most recent item as its final item.
"""
def __init__(self, _max_size, _random=None, always_keep_last=True):
"""Create the _ReservoirBucket.
Args:
_max_size: The maximum size the reservoir bucket may grow to. If size is
zero, the bucket has unbounded size.
_random: The random number generator to use. If not specified, defaults to
random.Random(0).
always_keep_last: Whether the latest seen item should always be included
in the end of the bucket.
Raises:
ValueError: if the size is not a nonnegative integer.
"""
if _max_size < 0 or _max_size != round(_max_size):
raise ValueError(
"_max_size must be nonnegative int, was %s" % _max_size
)
self.items = []
# This mutex protects the internal items, ensuring that calls to Items and
# AddItem are thread-safe
self._mutex = threading.Lock()
self._max_size = _max_size
self._num_items_seen = 0
if _random is not None:
self._random = _random
else:
self._random = random.Random(0)
self.always_keep_last = always_keep_last
def AddItem(self, item, f=lambda x: x):
"""Add an item to the ReservoirBucket, replacing an old item if
necessary.
The new item is guaranteed to be added to the bucket, and to be the last
element in the bucket. If the bucket has reached capacity, then an old item
will be replaced. With probability (_max_size/_num_items_seen) a random item
in the bucket will be popped out and the new item will be appended
to the end. With probability (1 - _max_size/_num_items_seen)
the last item in the bucket will be replaced.
Since the O(n) replacements occur with O(1/_num_items_seen) likelihood,
the amortized runtime is O(1).
Args:
item: The item to add to the bucket.
f: A function to transform item before addition, if it will be kept in
the reservoir.
"""
with self._mutex:
if len(self.items) < self._max_size or self._max_size == 0:
self.items.append(f(item))
else:
r = self._random.randint(0, self._num_items_seen)
if r < self._max_size:
self.items.pop(r)
self.items.append(f(item))
elif self.always_keep_last:
self.items[-1] = f(item)
self._num_items_seen += 1
def FilterItems(self, filterFn):
"""Filter items in a ReservoirBucket, using a filtering function.
Filtering items from the reservoir bucket must update the
internal state variable self._num_items_seen, which is used for determining
the rate of replacement in reservoir sampling. Ideally, self._num_items_seen
would contain the exact number of items that have ever seen by the
ReservoirBucket and satisfy filterFn. However, the ReservoirBucket does not
have access to all items seen -- it only has access to the subset of items
that have survived sampling (self.items). Therefore, we estimate
self._num_items_seen by scaling it by the same ratio as the ratio of items
not removed from self.items.
Args:
filterFn: A function that returns True for items to be kept.
Returns:
The number of items removed from the bucket.
"""
with self._mutex:
size_before = len(self.items)
self.items = list(filter(filterFn, self.items))
size_diff = size_before - len(self.items)
# Estimate a correction the number of items seen
prop_remaining = (
len(self.items) / float(size_before) if size_before > 0 else 0
)
self._num_items_seen = int(
round(self._num_items_seen * prop_remaining)
)
return size_diff
def Items(self):
"""Get all the items in the bucket."""
with self._mutex:
return list(self.items)
@@ -0,0 +1,29 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""String constants describing contents of an event accumulator."""
# Arbitrary strings chosen to pass the type information of the tag from
# the backend to the frontend.
TENSORS = "tensors"
GRAPH = "graph"
META_GRAPH = "meta_graph"
RUN_METADATA = "run_metadata"
# Legacy (pre-tensor) tag types.
COMPRESSED_HISTOGRAMS = "distributions"
HISTOGRAMS = "histograms"
IMAGES = "images"
AUDIO = "audio"
SCALARS = "scalars"
@@ -0,0 +1,73 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Application-level experiment ID support."""
import re
# Value of the first path component that signals that the second path
# component represents an experiment ID.
_EXPERIMENT_PATH_COMPONENT = "experiment"
# Key into the WSGI environment used for the experiment ID.
WSGI_ENVIRON_KEY = "HTTP_TENSORBOARD_EXPERIMENT_ID"
class ExperimentIdMiddleware:
"""WSGI middleware extracting experiment IDs from URL to environment.
Any request whose path matches `/experiment/SOME_EID[/...]` will have
its first two path components stripped, and its experiment ID stored
onto the WSGI environment with key taken from the `WSGI_ENVIRON_KEY`
constant. All other requests will have paths unchanged and the
experiment ID set to the empty string. It noops if the key taken from
the `WSGI_ENVIRON_KEY` is already present in the environment.
Instances of this class are WSGI applications (see PEP 3333).
"""
def __init__(self, application):
"""Initializes an `ExperimentIdMiddleware`.
Args:
application: The WSGI application to wrap (see PEP 3333).
"""
self._application = application
# Regular expression that matches the whole `/experiment/EID` prefix
# (without any trailing slash) and captures the experiment ID.
self._pat = re.compile(
r"/%s/([^/]*)" % re.escape(_EXPERIMENT_PATH_COMPONENT)
)
def __call__(self, environ, start_response):
# Skip ExperimentIdMiddleware was already called.
if WSGI_ENVIRON_KEY in environ:
return self._application(environ, start_response)
path = environ.get("PATH_INFO", "")
m = self._pat.match(path)
if m:
eid = m.group(1)
new_path = path[m.end(0) :]
root = m.group(0)
else:
eid = ""
new_path = path
root = ""
environ[WSGI_ENVIRON_KEY] = eid
environ["PATH_INFO"] = new_path
environ["SCRIPT_NAME"] = environ.get("SCRIPT_NAME", "") + root
return self._application(environ, start_response)
@@ -0,0 +1,51 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Experimental plugin support for TensorBoard.
Contains the mechanism for marking plugins as experimental.
"""
class ExperimentalPlugin:
"""A marker class used to annotate a plugin as experimental.
Experimental plugins are hidden from users by default. The plugin will only
be enabled for a user if the user has specified the plugin with the
experimentalPlugin query parameter in the URL.
The marker class can annotate either TBPlugin or TBLoader instances, whichever
is most convenient.
Typical usage is to create a new class that inherits from both an existing
TBPlugin/TBLoader class and this marker class. For example:
class ExperimentalGraphsPlugin(
graphs_plugin.GraphsPlugin,
experimental_plugin.ExperimentalPlugin,
):
pass
class ExperimentalDebuggerPluginLoader(
debugger_plugin_loader.DebuggerPluginLoader,
experimental_plugin.ExperimentalPlugin
):
pass
Note: This class is itself an experimental mechanism and is subject to
modification or removal without warning.
"""
pass
@@ -0,0 +1,264 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorBoard HTTP utilities."""
import gzip
import io
import json
import re
import struct
import time
import wsgiref.handlers
import werkzeug
from tensorboard.backend import json_util
_DISALLOWED_CHAR_IN_DOMAIN = re.compile(r"\s")
# TODO(stephanwlee): Refactor this to not use the module variable but
# instead use a configurable via some kind of assets provider which would
# hold configurations for the CSP.
# @vaadin/vaadin-lumo-styles/font-icons(via vaadin-grid) uses data URI for
# loading font icons.
_CSP_FONT_DOMAINS_WHITELIST = ["data:"]
_CSP_FRAME_DOMAINS_WHITELIST = []
_CSP_IMG_DOMAINS_WHITELIST = []
_CSP_SCRIPT_DOMAINS_WHITELIST = []
_CSP_CONNECT_DOMAINS_WHITELIST = []
_CSP_SCRIPT_SELF = True
# numericjs (via projector) uses unsafe-eval :(.
_CSP_SCRIPT_UNSAFE_EVAL = True
_CSP_STYLE_DOMAINS_WHITELIST = []
_EXTRACT_MIMETYPE_PATTERN = re.compile(r"^[^;\s]*")
_EXTRACT_CHARSET_PATTERN = re.compile(r"charset=([-_0-9A-Za-z]+)")
# Allows *, gzip or x-gzip, but forbid gzip;q=0
# https://tools.ietf.org/html/rfc7231#section-5.3.4
_ALLOWS_GZIP_PATTERN = re.compile(
r"(?:^|,|\s)(?:(?:x-)?gzip|\*)(?!;q=0)(?:\s|,|$)"
)
_TEXTUAL_MIMETYPES = set(
[
"application/javascript",
"application/json",
"application/json+protobuf",
"image/svg+xml",
"text/css",
"text/csv",
"text/html",
"text/javascript",
"text/plain",
"text/tab-separated-values",
"text/x-protobuf",
]
)
_JSON_MIMETYPES = set(
[
"application/json",
"application/json+protobuf",
]
)
# Do not support xhtml for now.
_HTML_MIMETYPE = "text/html"
def Respond(
request,
content,
content_type,
code=200,
expires=0,
content_encoding=None,
encoding="utf-8",
csp_scripts_sha256s=None,
headers=None,
):
"""Construct a werkzeug Response.
Responses are transmitted to the browser with compression if: a) the browser
supports it; b) it's sane to compress the content_type in question; and c)
the content isn't already compressed, as indicated by the content_encoding
parameter.
Browser and proxy caching is completely disabled by default. If the expires
parameter is greater than zero then the response will be able to be cached by
the browser for that many seconds; however, proxies are still forbidden from
caching so that developers can bypass the cache with Ctrl+Shift+R.
For textual content that isn't JSON, the encoding parameter is used as the
transmission charset which is automatically appended to the Content-Type
header. That is unless of course the content_type parameter contains a
charset parameter. If the two disagree, the characters in content will be
transcoded to the latter.
If content_type declares a JSON media type, then content MAY be a dict, list,
tuple, or set, in which case this function has an implicit composition with
json_util.Cleanse and json.dumps. The encoding parameter is used to decode
byte strings within the JSON object; therefore transmitting binary data
within JSON is not permitted. JSON is transmitted as ASCII unless the
content_type parameter explicitly defines a charset parameter, in which case
the serialized JSON bytes will use that instead of escape sequences.
Args:
request: A werkzeug Request object. Used mostly to check the
Accept-Encoding header.
content: Payload data as byte string, unicode string, or maybe JSON.
content_type: Media type and optionally an output charset.
code: Numeric HTTP status code to use.
expires: Second duration for browser caching.
content_encoding: Encoding if content is already encoded, e.g. 'gzip'.
encoding: Input charset if content parameter has byte strings.
csp_scripts_sha256s: List of base64 serialized sha256 of whitelisted script
elements for script-src of the Content-Security-Policy. If it is None, the
HTML will disallow any script to execute. It is only be used when the
content_type is text/html.
headers: Any additional headers to include on the response, as a
list of key-value tuples: e.g., `[("Allow", "GET")]`. In case of
conflict, these may be overridden with headers added by this function.
Returns:
A werkzeug Response object (a WSGI application).
"""
mimetype = _EXTRACT_MIMETYPE_PATTERN.search(content_type).group(0)
charset_match = _EXTRACT_CHARSET_PATTERN.search(content_type)
charset = charset_match.group(1) if charset_match else encoding
textual = charset_match or mimetype in _TEXTUAL_MIMETYPES
if mimetype in _JSON_MIMETYPES and isinstance(
content, (dict, list, set, tuple)
):
content = json.dumps(
json_util.Cleanse(content, encoding), ensure_ascii=not charset_match
)
# Ensure correct output encoding, transcoding if necessary.
if charset != encoding and isinstance(content, bytes):
content = content.decode(encoding)
if isinstance(content, str):
content = content.encode(charset)
if textual and not charset_match and mimetype not in _JSON_MIMETYPES:
content_type += "; charset=" + charset
gzip_accepted = _ALLOWS_GZIP_PATTERN.search(
request.headers.get("Accept-Encoding", "")
)
# Automatically gzip uncompressed text data if accepted.
if textual and not content_encoding and gzip_accepted:
out = io.BytesIO()
# Set mtime to zero to make payload for a given input deterministic.
with gzip.GzipFile(
fileobj=out, mode="wb", compresslevel=3, mtime=0
) as f:
f.write(content)
content = out.getvalue()
content_encoding = "gzip"
content_length = len(content)
direct_passthrough = False
# Automatically streamwise-gunzip precompressed data if not accepted.
if content_encoding == "gzip" and not gzip_accepted:
gzip_file = gzip.GzipFile(fileobj=io.BytesIO(content), mode="rb")
# Last 4 bytes of gzip formatted data (little-endian) store the original
# content length mod 2^32; we just assume it's the content length. That
# means we can't streamwise-gunzip >4 GB precompressed file; this is ok.
content_length = struct.unpack("<I", content[-4:])[0]
content = werkzeug.wsgi.wrap_file(request.environ, gzip_file)
content_encoding = None
direct_passthrough = True
headers = list(headers or [])
headers.append(("Content-Length", str(content_length)))
headers.append(("X-Content-Type-Options", "nosniff"))
if content_encoding:
headers.append(("Content-Encoding", content_encoding))
if expires > 0:
e = wsgiref.handlers.format_date_time(time.time() + float(expires))
headers.append(("Expires", e))
headers.append(("Cache-Control", "private, max-age=%d" % expires))
else:
headers.append(("Expires", "0"))
headers.append(("Cache-Control", "no-cache, must-revalidate"))
if mimetype == _HTML_MIMETYPE:
frags = (
_CSP_SCRIPT_DOMAINS_WHITELIST
+ [
"'self'" if _CSP_SCRIPT_SELF else "",
"'unsafe-eval'" if _CSP_SCRIPT_UNSAFE_EVAL else "",
]
+ [
"'sha256-{}'".format(sha256)
for sha256 in (csp_scripts_sha256s or [])
]
)
script_srcs = _create_csp_string(*frags)
csp_string = ";".join(
[
"default-src 'self'",
"font-src %s"
% _create_csp_string("'self'", *_CSP_FONT_DOMAINS_WHITELIST),
# Dynamic plugins are rendered inside an iframe.
"frame-src %s"
% _create_csp_string("'self'", *_CSP_FRAME_DOMAINS_WHITELIST),
"img-src %s"
% _create_csp_string(
"'self'",
# used by favicon
"data:",
# used by What-If tool for image sprites.
"blob:",
*_CSP_IMG_DOMAINS_WHITELIST,
),
"object-src 'none'",
"style-src %s"
% _create_csp_string(
"'self'",
# used by google-chart
"https://www.gstatic.com",
"data:",
# inline styles: Polymer templates + d3 uses inline styles.
"'unsafe-inline'",
*_CSP_STYLE_DOMAINS_WHITELIST,
),
"connect-src %s"
% _create_csp_string("'self'", *_CSP_CONNECT_DOMAINS_WHITELIST),
"script-src %s" % script_srcs,
]
)
headers.append(("Content-Security-Policy", csp_string))
if request.method == "HEAD":
content = None
return werkzeug.wrappers.Response(
response=content,
status=code,
headers=headers,
content_type=content_type,
direct_passthrough=direct_passthrough,
)
def _create_csp_string(*csp_fragments):
csp_string = " ".join([frag for frag in csp_fragments if frag])
return csp_string if csp_string else "'none'"
@@ -0,0 +1,72 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A module providing a function for serializing JSON values with Infinity.
Python provides no way to override how json.dumps serializes
Infinity/-Infinity/NaN; if allow_nan is true, it encodes them as
Infinity/-Infinity/NaN, in violation of the JSON spec and in violation
of what JSON.parse accepts. If it's false, it throws a ValueError,
Neither subclassing JSONEncoder nor passing a function in the |default|
keyword argument overrides this.
"""
import collections
import math
_INFINITY = float("inf")
_NEGATIVE_INFINITY = float("-inf")
def Cleanse(obj, encoding="utf-8"):
"""Makes Python object appropriate for JSON serialization.
- Replaces instances of Infinity/-Infinity/NaN with strings.
- Turns byte strings into unicode strings.
- Turns sets into sorted lists.
- Turns tuples into lists.
Args:
obj: Python data structure.
encoding: Charset used to decode byte strings.
Returns:
Unicode JSON data structure.
"""
if isinstance(obj, int):
return obj
elif isinstance(obj, float):
if obj == _INFINITY:
return "Infinity"
elif obj == _NEGATIVE_INFINITY:
return "-Infinity"
elif math.isnan(obj):
return "NaN"
else:
return obj
elif isinstance(obj, bytes):
return obj.decode(encoding)
elif isinstance(obj, (list, tuple)):
return [Cleanse(i, encoding) for i in obj]
elif isinstance(obj, set):
return [Cleanse(i, encoding) for i in sorted(obj)]
elif isinstance(obj, dict):
return collections.OrderedDict(
(Cleanse(k, encoding), Cleanse(v, encoding)) for k, v in obj.items()
)
else:
return obj
@@ -0,0 +1,68 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Internal path prefix support for TensorBoard.
Using a path prefix of `/foo/bar` enables TensorBoard to serve from
`http://localhost:6006/foo/bar/` rather than `http://localhost:6006/`.
See the `--path_prefix` flag docs for more details.
"""
from tensorboard import errors
class PathPrefixMiddleware:
"""WSGI middleware for path prefixes.
All requests to this middleware must begin with the specified path
prefix (otherwise, a 404 will be returned immediately). Requests
will be forwarded to the underlying application with the path prefix
stripped and appended to `SCRIPT_NAME` (see the WSGI spec, PEP 3333,
for details).
"""
def __init__(self, application, path_prefix):
"""Initializes this middleware.
Args:
application: The WSGI application to wrap (see PEP 3333).
path_prefix: A string path prefix to be stripped from incoming
requests. If empty, this middleware is a no-op. If non-empty,
the path prefix must start with a slash and not end with one
(e.g., "/tensorboard").
"""
if path_prefix.endswith("/"):
raise ValueError(
"Path prefix must not end with slash: %r" % path_prefix
)
if path_prefix and not path_prefix.startswith("/"):
raise ValueError(
"Non-empty path prefix must start with slash: %r" % path_prefix
)
self._application = application
self._path_prefix = path_prefix
self._strict_prefix = self._path_prefix + "/"
def __call__(self, environ, start_response):
path = environ.get("PATH_INFO", "")
if path != self._path_prefix and not path.startswith(
self._strict_prefix
):
raise errors.NotFoundError()
environ["PATH_INFO"] = path[len(self._path_prefix) :]
environ["SCRIPT_NAME"] = (
environ.get("SCRIPT_NAME", "") + self._path_prefix
)
return self._application(environ, start_response)
@@ -0,0 +1,74 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Graph post-processing logic.
Used by both TensorBoard and mldash.
"""
def prepare_graph_for_ui(
graph, limit_attr_size=1024, large_attrs_key="_too_large_attrs"
):
"""Prepares (modifies in-place) the graph to be served to the front-end.
For now, it supports filtering out attributes that are
too large to be shown in the graph UI.
Args:
graph: The GraphDef proto message.
limit_attr_size: Maximum allowed size in bytes, before the attribute
is considered large. Default is 1024 (1KB). Must be > 0 or None.
If None, there will be no filtering.
large_attrs_key: The attribute key that will be used for storing attributes
that are too large. Default is '_too_large_attrs'. Must be != None if
`limit_attr_size` is != None.
Raises:
ValueError: If `large_attrs_key is None` while `limit_attr_size != None`.
ValueError: If `limit_attr_size` is defined, but <= 0.
"""
# TODO(@davidsoergel): detect whether a graph has been filtered already
# (to a limit_attr_size <= what is requested here). If it is already
# filtered, return immediately.
# Check input for validity.
if limit_attr_size is not None:
if large_attrs_key is None:
raise ValueError(
"large_attrs_key must be != None when limit_attr_size"
"!= None."
)
if limit_attr_size <= 0:
raise ValueError(
"limit_attr_size must be > 0, but is %d" % limit_attr_size
)
# Filter only if a limit size is defined.
if limit_attr_size is not None:
for node in graph.node:
# Go through all the attributes and filter out ones bigger than the
# limit.
keys = list(node.attr.keys())
for key in keys:
size = node.attr[key].ByteSize()
if size > limit_attr_size or size < 0:
del node.attr[key]
# Add the attribute key to the list of "too large" attributes.
# This is used in the info card in the graph UI to show the user
# that some attributes are too large to be shown.
node.attr[large_attrs_key].list.s.append(
key.encode("utf-8")
)
@@ -0,0 +1,203 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Validates responses and their security features."""
import dataclasses
from typing import Collection
from werkzeug.datastructures import Headers
from werkzeug import http
from tensorboard.util import tb_logging
logger = tb_logging.get_logger()
_HTML_MIME_TYPE = "text/html"
_CSP_DEFAULT_SRC = "default-src"
# Whitelist of allowed CSP violations.
_CSP_IGNORE = {
# Polymer-based code uses unsafe-inline.
"style-src": ["'unsafe-inline'", "data:"],
# Used in canvas
"img-src": ["blob:", "data:"],
# Used by numericjs.
# TODO(stephanwlee): remove it eventually.
"script-src": ["'unsafe-eval'"],
"font-src": ["data:"],
}
@dataclasses.dataclass(frozen=True)
class Directive:
"""Content security policy directive.
Loosely follow vocabulary from https://www.w3.org/TR/CSP/#framework-directives.
Attributes:
name: A non-empty string.
value: A collection of non-empty strings.
"""
name: str
value: Collection[str]
def _maybe_raise_value_error(error_msg):
logger.warning("In 3.0, this warning will become an error:\n%s" % error_msg)
# TODO(3.x): raise a value error.
class SecurityValidatorMiddleware:
"""WSGI middleware validating security on response.
It validates:
- responses have Content-Type
- responses have X-Content-Type-Options: nosniff
- text/html responses have CSP header. It also validates whether the CSP
headers pass basic requirement. e.g., default-src should be present, cannot
use "*" directive, and others. For more complete list, please refer to
_validate_csp_policies.
Instances of this class are WSGI applications (see PEP 3333).
"""
def __init__(self, application):
"""Initializes an `SecurityValidatorMiddleware`.
Args:
application: The WSGI application to wrap (see PEP 3333).
"""
self._application = application
def __call__(self, environ, start_response):
def start_response_proxy(status, headers, exc_info=None):
self._validate_headers(headers)
return start_response(status, headers, exc_info)
return self._application(environ, start_response_proxy)
def _validate_headers(self, headers_list):
headers = Headers(headers_list)
self._validate_content_type(headers)
self._validate_x_content_type_options(headers)
self._validate_csp_headers(headers)
def _validate_content_type(self, headers):
if headers.get("Content-Type"):
return
_maybe_raise_value_error("Content-Type is required on a Response")
def _validate_x_content_type_options(self, headers):
option = headers.get("X-Content-Type-Options")
if option == "nosniff":
return
_maybe_raise_value_error(
'X-Content-Type-Options is required to be "nosniff"'
)
def _validate_csp_headers(self, headers):
mime_type, _ = http.parse_options_header(headers.get("Content-Type"))
if mime_type != _HTML_MIME_TYPE:
return
csp_texts = headers.get_all("Content-Security-Policy")
policies = []
for csp_text in csp_texts:
policies += self._parse_serialized_csp(csp_text)
self._validate_csp_policies(policies)
def _validate_csp_policies(self, policies):
has_default_src = False
violations = []
for directive in policies:
name = directive.name
for value in directive.value:
has_default_src = has_default_src or name == _CSP_DEFAULT_SRC
if value in _CSP_IGNORE.get(name, []):
# There are cases where certain directives are legitimate.
continue
# TensorBoard follows principle of least privilege. However, to make it
# easier to conform to the security policy for plugin authors,
# TensorBoard trusts request and resources originating its server. Also,
# it can selectively trust domains as long as they use https protocol.
# Lastly, it can allow 'none' directive.
# TODO(stephanwlee): allow configuration for whitelist of domains for
# stricter enforcement.
# TODO(stephanwlee): deprecate the sha-based whitelisting.
if (
value == "'self'"
or value == "'none'"
or value.startswith("https:")
or value.startswith("'sha256-")
):
continue
msg = "Illegal Content-Security-Policy for {name}: {value}".format(
name=name, value=value
)
violations.append(msg)
if not has_default_src:
violations.append(
"Requires default-src for Content-Security-Policy"
)
if violations:
_maybe_raise_value_error("\n".join(violations))
def _parse_serialized_csp(self, csp_text):
# See https://www.w3.org/TR/CSP/#parse-serialized-policy.
# Below Steps are based on the algorithm stated in above spec.
# Deviations:
# - it does not warn and ignore duplicative directive (Step 2.5)
# Step 2
csp_srcs = csp_text.split(";")
policy = []
for token in csp_srcs:
# Step 2.1
token = token.strip()
if not token:
# Step 2.2
continue
# Step 2.3
token_frag = token.split(None, 1)
name = token_frag[0]
values = token_frag[1] if len(token_frag) == 2 else ""
# Step 2.4
name = name.lower()
# Step 2.6
value = values.split()
# Step 2.7
directive = Directive(name=name, value=value)
# Step 2.8
policy.append(directive)
return policy
@@ -0,0 +1,70 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compatibility interfaces for TensorBoard.
This module provides logic for importing variations on the TensorFlow
APIs, as lazily loaded imports to help avoid circular dependency issues
and defer the search and loading of the module until necessary.
"""
import tensorboard.lazy as _lazy
@_lazy.lazy_load("tensorboard.compat.tf")
def tf():
"""Provide the root module of a TF-like API for use within TensorBoard.
By default this is equivalent to `import tensorflow as tf`, but it can be used
in combination with //tensorboard/compat:tensorflow (to fall back to a stub TF
API implementation if the real one is not available) or with
//tensorboard/compat:no_tensorflow (to force unconditional use of the stub).
Returns:
The root module of a TF-like API, if available.
Raises:
ImportError: if a TF-like API is not available.
"""
try:
from tensorboard.compat import notf # noqa: F401
except ImportError:
try:
import tensorflow
return tensorflow
except ImportError:
pass
from tensorboard.compat import tensorflow_stub
return tensorflow_stub
@_lazy.lazy_load("tensorboard.compat.tf2")
def tf2():
"""Provide the root module of a TF-2.0 API for use within TensorBoard.
Returns:
The root module of a TF-2.0 API, if available.
Raises:
ImportError: if a TF-2.0 API is not available.
"""
# Resolve the lazy `tf` compat API from earlier in this file and try to find
# tf.compat.v2. Don't check tf.__version__ since this is not always reliable
# if TF was built with tf_api_version!=2.
if hasattr(tf, "compat") and hasattr(tf.compat, "v2"):
return tf.compat.v2
raise ImportError("cannot import tensorflow 2.0 API")
@@ -0,0 +1,35 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/allocation_description.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5tensorboard/compat/proto/allocation_description.proto\x12\x0btensorboard\"\xa3\x01\n\x15\x41llocationDescription\x12\x17\n\x0frequested_bytes\x18\x01 \x01(\x03\x12\x17\n\x0f\x61llocated_bytes\x18\x02 \x01(\x03\x12\x16\n\x0e\x61llocator_name\x18\x03 \x01(\t\x12\x15\n\rallocation_id\x18\x04 \x01(\x03\x12\x1c\n\x14has_single_reference\x18\x05 \x01(\x08\x12\x0b\n\x03ptr\x18\x06 \x01(\x04\x42\x9b\x01\n\x18org.tensorflow.frameworkB\x1b\x41llocationDescriptionProtosP\x01Z]github.com/tensorflow/tensorflow/tensorflow/go/core/framework/allocation_description_go_proto\xf8\x01\x01\x62\x06proto3')
_ALLOCATIONDESCRIPTION = DESCRIPTOR.message_types_by_name['AllocationDescription']
AllocationDescription = _reflection.GeneratedProtocolMessageType('AllocationDescription', (_message.Message,), {
'DESCRIPTOR' : _ALLOCATIONDESCRIPTION,
'__module__' : 'tensorboard.compat.proto.allocation_description_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.AllocationDescription)
})
_sym_db.RegisterMessage(AllocationDescription)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\033AllocationDescriptionProtosP\001Z]github.com/tensorflow/tensorflow/tensorflow/go/core/framework/allocation_description_go_proto\370\001\001'
_ALLOCATIONDESCRIPTION._serialized_start=71
_ALLOCATIONDESCRIPTION._serialized_end=234
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/api_def.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import attr_value_pb2 as tensorboard_dot_compat_dot_proto_dot_attr__value__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&tensorboard/compat/proto/api_def.proto\x12\x0btensorboard\x1a)tensorboard/compat/proto/attr_value.proto\"\xe7\x05\n\x06\x41piDef\x12\x15\n\rgraph_op_name\x18\x01 \x01(\t\x12\x1b\n\x13\x64\x65precation_message\x18\x0c \x01(\t\x12\x1b\n\x13\x64\x65precation_version\x18\r \x01(\x05\x12\x32\n\nvisibility\x18\x02 \x01(\x0e\x32\x1e.tensorboard.ApiDef.Visibility\x12.\n\x08\x65ndpoint\x18\x03 \x03(\x0b\x32\x1c.tensorboard.ApiDef.Endpoint\x12\'\n\x06in_arg\x18\x04 \x03(\x0b\x32\x17.tensorboard.ApiDef.Arg\x12(\n\x07out_arg\x18\x05 \x03(\x0b\x32\x17.tensorboard.ApiDef.Arg\x12\x11\n\targ_order\x18\x0b \x03(\t\x12&\n\x04\x61ttr\x18\x06 \x03(\x0b\x32\x18.tensorboard.ApiDef.Attr\x12\x0f\n\x07summary\x18\x07 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x08 \x01(\t\x12\x1a\n\x12\x64\x65scription_prefix\x18\t \x01(\t\x12\x1a\n\x12\x64\x65scription_suffix\x18\n \x01(\t\x1aI\n\x08\x45ndpoint\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x03 \x01(\x08\x12\x1b\n\x13\x64\x65precation_version\x18\x04 \x01(\x05\x1a;\n\x03\x41rg\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\trename_to\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x1ak\n\x04\x41ttr\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\trename_to\x18\x02 \x01(\t\x12-\n\rdefault_value\x18\x03 \x01(\x0b\x32\x16.tensorboard.AttrValue\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\"G\n\nVisibility\x12\x16\n\x12\x44\x45\x46\x41ULT_VISIBILITY\x10\x00\x12\x0b\n\x07VISIBLE\x10\x01\x12\x08\n\x04SKIP\x10\x02\x12\n\n\x06HIDDEN\x10\x03\"*\n\x07\x41piDefs\x12\x1f\n\x02op\x18\x01 \x03(\x0b\x32\x13.tensorboard.ApiDefB}\n\x18org.tensorflow.frameworkB\x0c\x41piDefProtosP\x01ZNgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/api_def_go_proto\xf8\x01\x01\x62\x06proto3')
_APIDEF = DESCRIPTOR.message_types_by_name['ApiDef']
_APIDEF_ENDPOINT = _APIDEF.nested_types_by_name['Endpoint']
_APIDEF_ARG = _APIDEF.nested_types_by_name['Arg']
_APIDEF_ATTR = _APIDEF.nested_types_by_name['Attr']
_APIDEFS = DESCRIPTOR.message_types_by_name['ApiDefs']
_APIDEF_VISIBILITY = _APIDEF.enum_types_by_name['Visibility']
ApiDef = _reflection.GeneratedProtocolMessageType('ApiDef', (_message.Message,), {
'Endpoint' : _reflection.GeneratedProtocolMessageType('Endpoint', (_message.Message,), {
'DESCRIPTOR' : _APIDEF_ENDPOINT,
'__module__' : 'tensorboard.compat.proto.api_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ApiDef.Endpoint)
})
,
'Arg' : _reflection.GeneratedProtocolMessageType('Arg', (_message.Message,), {
'DESCRIPTOR' : _APIDEF_ARG,
'__module__' : 'tensorboard.compat.proto.api_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ApiDef.Arg)
})
,
'Attr' : _reflection.GeneratedProtocolMessageType('Attr', (_message.Message,), {
'DESCRIPTOR' : _APIDEF_ATTR,
'__module__' : 'tensorboard.compat.proto.api_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ApiDef.Attr)
})
,
'DESCRIPTOR' : _APIDEF,
'__module__' : 'tensorboard.compat.proto.api_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ApiDef)
})
_sym_db.RegisterMessage(ApiDef)
_sym_db.RegisterMessage(ApiDef.Endpoint)
_sym_db.RegisterMessage(ApiDef.Arg)
_sym_db.RegisterMessage(ApiDef.Attr)
ApiDefs = _reflection.GeneratedProtocolMessageType('ApiDefs', (_message.Message,), {
'DESCRIPTOR' : _APIDEFS,
'__module__' : 'tensorboard.compat.proto.api_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ApiDefs)
})
_sym_db.RegisterMessage(ApiDefs)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\014ApiDefProtosP\001ZNgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/api_def_go_proto\370\001\001'
_APIDEF._serialized_start=99
_APIDEF._serialized_end=842
_APIDEF_ENDPOINT._serialized_start=526
_APIDEF_ENDPOINT._serialized_end=599
_APIDEF_ARG._serialized_start=601
_APIDEF_ARG._serialized_end=660
_APIDEF_ATTR._serialized_start=662
_APIDEF_ATTR._serialized_end=769
_APIDEF_VISIBILITY._serialized_start=771
_APIDEF_VISIBILITY._serialized_end=842
_APIDEFS._serialized_start=844
_APIDEFS._serialized_end=886
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/attr_value.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import tensor_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__pb2
from tensorboard.compat.proto import tensor_shape_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)tensorboard/compat/proto/attr_value.proto\x12\x0btensorboard\x1a%tensorboard/compat/proto/tensor.proto\x1a+tensorboard/compat/proto/tensor_shape.proto\x1a$tensorboard/compat/proto/types.proto\"\xaf\x04\n\tAttrValue\x12\x0b\n\x01s\x18\x02 \x01(\x0cH\x00\x12\x0b\n\x01i\x18\x03 \x01(\x03H\x00\x12\x0b\n\x01\x66\x18\x04 \x01(\x02H\x00\x12\x0b\n\x01\x62\x18\x05 \x01(\x08H\x00\x12%\n\x04type\x18\x06 \x01(\x0e\x32\x15.tensorboard.DataTypeH\x00\x12.\n\x05shape\x18\x07 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProtoH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.tensorboard.TensorProtoH\x00\x12\x30\n\x04list\x18\x01 \x01(\x0b\x32 .tensorboard.AttrValue.ListValueH\x00\x12)\n\x04\x66unc\x18\n \x01(\x0b\x32\x19.tensorboard.NameAttrListH\x00\x12\x15\n\x0bplaceholder\x18\t \x01(\tH\x00\x1a\xed\x01\n\tListValue\x12\t\n\x01s\x18\x02 \x03(\x0c\x12\r\n\x01i\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\r\n\x01\x66\x18\x04 \x03(\x02\x42\x02\x10\x01\x12\r\n\x01\x62\x18\x05 \x03(\x08\x42\x02\x10\x01\x12\'\n\x04type\x18\x06 \x03(\x0e\x32\x15.tensorboard.DataTypeB\x02\x10\x01\x12,\n\x05shape\x18\x07 \x03(\x0b\x32\x1d.tensorboard.TensorShapeProto\x12(\n\x06tensor\x18\x08 \x03(\x0b\x32\x18.tensorboard.TensorProto\x12\'\n\x04\x66unc\x18\t \x03(\x0b\x32\x19.tensorboard.NameAttrListB\x07\n\x05value\"\x94\x01\n\x0cNameAttrList\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x31\n\x04\x61ttr\x18\x02 \x03(\x0b\x32#.tensorboard.NameAttrList.AttrEntry\x1a\x43\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.tensorboard.AttrValue:\x02\x38\x01\x42\x83\x01\n\x18org.tensorflow.frameworkB\x0f\x41ttrValueProtosP\x01ZQgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/attr_value_go_proto\xf8\x01\x01\x62\x06proto3')
_ATTRVALUE = DESCRIPTOR.message_types_by_name['AttrValue']
_ATTRVALUE_LISTVALUE = _ATTRVALUE.nested_types_by_name['ListValue']
_NAMEATTRLIST = DESCRIPTOR.message_types_by_name['NameAttrList']
_NAMEATTRLIST_ATTRENTRY = _NAMEATTRLIST.nested_types_by_name['AttrEntry']
AttrValue = _reflection.GeneratedProtocolMessageType('AttrValue', (_message.Message,), {
'ListValue' : _reflection.GeneratedProtocolMessageType('ListValue', (_message.Message,), {
'DESCRIPTOR' : _ATTRVALUE_LISTVALUE,
'__module__' : 'tensorboard.compat.proto.attr_value_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.AttrValue.ListValue)
})
,
'DESCRIPTOR' : _ATTRVALUE,
'__module__' : 'tensorboard.compat.proto.attr_value_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.AttrValue)
})
_sym_db.RegisterMessage(AttrValue)
_sym_db.RegisterMessage(AttrValue.ListValue)
NameAttrList = _reflection.GeneratedProtocolMessageType('NameAttrList', (_message.Message,), {
'AttrEntry' : _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), {
'DESCRIPTOR' : _NAMEATTRLIST_ATTRENTRY,
'__module__' : 'tensorboard.compat.proto.attr_value_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.NameAttrList.AttrEntry)
})
,
'DESCRIPTOR' : _NAMEATTRLIST,
'__module__' : 'tensorboard.compat.proto.attr_value_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.NameAttrList)
})
_sym_db.RegisterMessage(NameAttrList)
_sym_db.RegisterMessage(NameAttrList.AttrEntry)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\017AttrValueProtosP\001ZQgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/attr_value_go_proto\370\001\001'
_ATTRVALUE_LISTVALUE.fields_by_name['i']._options = None
_ATTRVALUE_LISTVALUE.fields_by_name['i']._serialized_options = b'\020\001'
_ATTRVALUE_LISTVALUE.fields_by_name['f']._options = None
_ATTRVALUE_LISTVALUE.fields_by_name['f']._serialized_options = b'\020\001'
_ATTRVALUE_LISTVALUE.fields_by_name['b']._options = None
_ATTRVALUE_LISTVALUE.fields_by_name['b']._serialized_options = b'\020\001'
_ATTRVALUE_LISTVALUE.fields_by_name['type']._options = None
_ATTRVALUE_LISTVALUE.fields_by_name['type']._serialized_options = b'\020\001'
_NAMEATTRLIST_ATTRENTRY._options = None
_NAMEATTRLIST_ATTRENTRY._serialized_options = b'8\001'
_ATTRVALUE._serialized_start=181
_ATTRVALUE._serialized_end=740
_ATTRVALUE_LISTVALUE._serialized_start=494
_ATTRVALUE_LISTVALUE._serialized_end=731
_NAMEATTRLIST._serialized_start=743
_NAMEATTRLIST._serialized_end=891
_NAMEATTRLIST_ATTRENTRY._serialized_start=824
_NAMEATTRLIST_ATTRENTRY._serialized_end=891
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/cluster.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&tensorboard/compat/proto/cluster.proto\x12\x0btensorboard\"s\n\x06JobDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12-\n\x05tasks\x18\x02 \x03(\x0b\x32\x1e.tensorboard.JobDef.TasksEntry\x1a,\n\nTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\".\n\nClusterDef\x12 \n\x03job\x18\x01 \x03(\x0b\x32\x13.tensorboard.JobDefB\x87\x01\n\x1aorg.tensorflow.distruntimeB\rClusterProtosP\x01ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\xf8\x01\x01\x62\x06proto3')
_JOBDEF = DESCRIPTOR.message_types_by_name['JobDef']
_JOBDEF_TASKSENTRY = _JOBDEF.nested_types_by_name['TasksEntry']
_CLUSTERDEF = DESCRIPTOR.message_types_by_name['ClusterDef']
JobDef = _reflection.GeneratedProtocolMessageType('JobDef', (_message.Message,), {
'TasksEntry' : _reflection.GeneratedProtocolMessageType('TasksEntry', (_message.Message,), {
'DESCRIPTOR' : _JOBDEF_TASKSENTRY,
'__module__' : 'tensorboard.compat.proto.cluster_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.JobDef.TasksEntry)
})
,
'DESCRIPTOR' : _JOBDEF,
'__module__' : 'tensorboard.compat.proto.cluster_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.JobDef)
})
_sym_db.RegisterMessage(JobDef)
_sym_db.RegisterMessage(JobDef.TasksEntry)
ClusterDef = _reflection.GeneratedProtocolMessageType('ClusterDef', (_message.Message,), {
'DESCRIPTOR' : _CLUSTERDEF,
'__module__' : 'tensorboard.compat.proto.cluster_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ClusterDef)
})
_sym_db.RegisterMessage(ClusterDef)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\032org.tensorflow.distruntimeB\rClusterProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_JOBDEF_TASKSENTRY._options = None
_JOBDEF_TASKSENTRY._serialized_options = b'8\001'
_JOBDEF._serialized_start=55
_JOBDEF._serialized_end=170
_JOBDEF_TASKSENTRY._serialized_start=126
_JOBDEF_TASKSENTRY._serialized_end=170
_CLUSTERDEF._serialized_start=172
_CLUSTERDEF._serialized_end=218
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,271 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/config.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import coordination_config_pb2 as tensorboard_dot_compat_dot_proto_dot_coordination__config__pb2
from tensorboard.compat.proto import cost_graph_pb2 as tensorboard_dot_compat_dot_proto_dot_cost__graph__pb2
from tensorboard.compat.proto import graph_pb2 as tensorboard_dot_compat_dot_proto_dot_graph__pb2
from tensorboard.compat.proto import step_stats_pb2 as tensorboard_dot_compat_dot_proto_dot_step__stats__pb2
from tensorboard.compat.proto import cluster_pb2 as tensorboard_dot_compat_dot_proto_dot_cluster__pb2
from tensorboard.compat.proto import debug_pb2 as tensorboard_dot_compat_dot_proto_dot_debug__pb2
from tensorboard.compat.proto import rewriter_config_pb2 as tensorboard_dot_compat_dot_proto_dot_rewriter__config__pb2
from tensorboard.compat.proto import rpc_options_pb2 as tensorboard_dot_compat_dot_proto_dot_rpc__options__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%tensorboard/compat/proto/config.proto\x12\x0btensorboard\x1a\x32tensorboard/compat/proto/coordination_config.proto\x1a)tensorboard/compat/proto/cost_graph.proto\x1a$tensorboard/compat/proto/graph.proto\x1a)tensorboard/compat/proto/step_stats.proto\x1a&tensorboard/compat/proto/cluster.proto\x1a$tensorboard/compat/proto/debug.proto\x1a.tensorboard/compat/proto/rewriter_config.proto\x1a*tensorboard/compat/proto/rpc_options.proto\"\x8c\n\n\nGPUOptions\x12\'\n\x1fper_process_gpu_memory_fraction\x18\x01 \x01(\x01\x12\x14\n\x0c\x61llow_growth\x18\x04 \x01(\x08\x12\x16\n\x0e\x61llocator_type\x18\x02 \x01(\t\x12\x1f\n\x17\x64\x65\x66\x65rred_deletion_bytes\x18\x03 \x01(\x03\x12\x1b\n\x13visible_device_list\x18\x05 \x01(\t\x12\"\n\x1apolling_active_delay_usecs\x18\x06 \x01(\x05\x12$\n\x1cpolling_inactive_delay_msecs\x18\x07 \x01(\x05\x12\x1c\n\x14\x66orce_gpu_compatible\x18\x08 \x01(\x08\x12:\n\x0c\x65xperimental\x18\t \x01(\x0b\x32$.tensorboard.GPUOptions.Experimental\x1a\xc4\x07\n\x0c\x45xperimental\x12L\n\x0fvirtual_devices\x18\x01 \x03(\x0b\x32\x33.tensorboard.GPUOptions.Experimental.VirtualDevices\x12#\n\x1bnum_virtual_devices_per_gpu\x18\x0f \x01(\x05\x12\x1a\n\x12use_unified_memory\x18\x02 \x01(\x08\x12#\n\x1bnum_dev_to_dev_copy_streams\x18\x03 \x01(\x05\x12\x1d\n\x15\x63ollective_ring_order\x18\x04 \x01(\t\x12\x1d\n\x15timestamped_allocator\x18\x05 \x01(\x08\x12#\n\x1bkernel_tracker_max_interval\x18\x07 \x01(\x05\x12 \n\x18kernel_tracker_max_bytes\x18\x08 \x01(\x05\x12\"\n\x1akernel_tracker_max_pending\x18\t \x01(\x05\x12\'\n\x1finternal_fragmentation_fraction\x18\n \x01(\x01\x12\x1d\n\x15use_cuda_malloc_async\x18\x0b \x01(\x08\x12,\n$disallow_retry_on_allocation_failure\x18\x0c \x01(\x08\x12 \n\x18gpu_host_mem_limit_in_mb\x18\r \x01(\x02\x12$\n\x1cgpu_host_mem_disallow_growth\x18\x0e \x01(\x08\x12$\n\x1cgpu_system_memory_size_in_mb\x18\x10 \x01(\x05\x12.\n&populate_pjrt_gpu_client_creation_info\x18\x11 \x01(\x08\x12\x0f\n\x07node_id\x18\x12 \x01(\x05\x12U\n\x14stream_merge_options\x18\x13 \x01(\x0b\x32\x37.tensorboard.GPUOptions.Experimental.StreamMergeOptions\x1aS\n\x0eVirtualDevices\x12\x17\n\x0fmemory_limit_mb\x18\x01 \x03(\x02\x12\x10\n\x08priority\x18\x02 \x03(\x05\x12\x16\n\x0e\x64\x65vice_ordinal\x18\x03 \x03(\x05\x1a\x85\x01\n\x12StreamMergeOptions\x12#\n\x1bmerge_host_to_device_stream\x18\x01 \x01(\x08\x12#\n\x1bmerge_device_to_host_stream\x18\x02 \x01(\x08\x12%\n\x1dmerge_device_to_device_stream\x18\x03 \x01(\x08\"\x9f\x03\n\x10OptimizerOptions\x12+\n#do_common_subexpression_elimination\x18\x01 \x01(\x08\x12\x1b\n\x13\x64o_constant_folding\x18\x02 \x01(\x08\x12$\n\x1cmax_folded_constant_in_bytes\x18\x06 \x01(\x03\x12\x1c\n\x14\x64o_function_inlining\x18\x04 \x01(\x08\x12\x36\n\topt_level\x18\x03 \x01(\x0e\x32#.tensorboard.OptimizerOptions.Level\x12\x46\n\x10global_jit_level\x18\x05 \x01(\x0e\x32,.tensorboard.OptimizerOptions.GlobalJitLevel\x12\x16\n\x0e\x63pu_global_jit\x18\x07 \x01(\x08\" \n\x05Level\x12\x06\n\x02L1\x10\x00\x12\x0f\n\x02L0\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\"C\n\x0eGlobalJitLevel\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x10\n\x03OFF\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x12\x08\n\x04ON_1\x10\x01\x12\x08\n\x04ON_2\x10\x02\"\xf0\x02\n\x0cGraphOptions\x12\x1e\n\x16\x65nable_recv_scheduling\x18\x02 \x01(\x08\x12\x38\n\x11optimizer_options\x18\x03 \x01(\x0b\x32\x1d.tensorboard.OptimizerOptions\x12\x18\n\x10\x62uild_cost_model\x18\x04 \x01(\x03\x12\x1e\n\x16\x62uild_cost_model_after\x18\t \x01(\x03\x12\x14\n\x0cinfer_shapes\x18\x05 \x01(\x08\x12\x1a\n\x12place_pruned_graph\x18\x06 \x01(\x08\x12 \n\x18\x65nable_bfloat16_sendrecv\x18\x07 \x01(\x08\x12\x15\n\rtimeline_step\x18\x08 \x01(\x05\x12\x34\n\x0frewrite_options\x18\n \x01(\x0b\x32\x1b.tensorboard.RewriterConfigJ\x04\x08\x01\x10\x02R%skip_common_subexpression_elimination\"A\n\x15ThreadPoolOptionProto\x12\x13\n\x0bnum_threads\x18\x01 \x01(\x05\x12\x13\n\x0bglobal_name\x18\x02 \x01(\t\"0\n\x0fSessionMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x03\"\xbf\x11\n\x0b\x43onfigProto\x12?\n\x0c\x64\x65vice_count\x18\x01 \x03(\x0b\x32).tensorboard.ConfigProto.DeviceCountEntry\x12$\n\x1cintra_op_parallelism_threads\x18\x02 \x01(\x05\x12$\n\x1cinter_op_parallelism_threads\x18\x05 \x01(\x05\x12\x1f\n\x17use_per_session_threads\x18\t \x01(\x08\x12H\n\x1csession_inter_op_thread_pool\x18\x0c \x03(\x0b\x32\".tensorboard.ThreadPoolOptionProto\x12\x18\n\x10placement_period\x18\x03 \x01(\x05\x12\x16\n\x0e\x64\x65vice_filters\x18\x04 \x03(\t\x12,\n\x0bgpu_options\x18\x06 \x01(\x0b\x32\x17.tensorboard.GPUOptions\x12\x39\n\x18pluggable_device_options\x18\x12 \x01(\x0b\x32\x17.tensorboard.GPUOptions\x12\x1c\n\x14\x61llow_soft_placement\x18\x07 \x01(\x08\x12\x1c\n\x14log_device_placement\x18\x08 \x01(\x08\x12\x30\n\rgraph_options\x18\n \x01(\x0b\x32\x19.tensorboard.GraphOptions\x12\x1f\n\x17operation_timeout_in_ms\x18\x0b \x01(\x03\x12,\n\x0brLine truncated
_GPUOPTIONS = DESCRIPTOR.message_types_by_name['GPUOptions']
_GPUOPTIONS_EXPERIMENTAL = _GPUOPTIONS.nested_types_by_name['Experimental']
_GPUOPTIONS_EXPERIMENTAL_VIRTUALDEVICES = _GPUOPTIONS_EXPERIMENTAL.nested_types_by_name['VirtualDevices']
_GPUOPTIONS_EXPERIMENTAL_STREAMMERGEOPTIONS = _GPUOPTIONS_EXPERIMENTAL.nested_types_by_name['StreamMergeOptions']
_OPTIMIZEROPTIONS = DESCRIPTOR.message_types_by_name['OptimizerOptions']
_GRAPHOPTIONS = DESCRIPTOR.message_types_by_name['GraphOptions']
_THREADPOOLOPTIONPROTO = DESCRIPTOR.message_types_by_name['ThreadPoolOptionProto']
_SESSIONMETADATA = DESCRIPTOR.message_types_by_name['SessionMetadata']
_CONFIGPROTO = DESCRIPTOR.message_types_by_name['ConfigProto']
_CONFIGPROTO_DEVICECOUNTENTRY = _CONFIGPROTO.nested_types_by_name['DeviceCountEntry']
_CONFIGPROTO_EXPERIMENTAL = _CONFIGPROTO.nested_types_by_name['Experimental']
_RUNOPTIONS = DESCRIPTOR.message_types_by_name['RunOptions']
_RUNOPTIONS_EXPERIMENTAL = _RUNOPTIONS.nested_types_by_name['Experimental']
_RUNOPTIONS_EXPERIMENTAL_RUNHANDLERPOOLOPTIONS = _RUNOPTIONS_EXPERIMENTAL.nested_types_by_name['RunHandlerPoolOptions']
_RUNMETADATA = DESCRIPTOR.message_types_by_name['RunMetadata']
_RUNMETADATA_FUNCTIONGRAPHS = _RUNMETADATA.nested_types_by_name['FunctionGraphs']
_TENSORCONNECTION = DESCRIPTOR.message_types_by_name['TensorConnection']
_CALLABLEOPTIONS = DESCRIPTOR.message_types_by_name['CallableOptions']
_CALLABLEOPTIONS_FEEDDEVICESENTRY = _CALLABLEOPTIONS.nested_types_by_name['FeedDevicesEntry']
_CALLABLEOPTIONS_FETCHDEVICESENTRY = _CALLABLEOPTIONS.nested_types_by_name['FetchDevicesEntry']
_BATCHINGOPTIONS = DESCRIPTOR.message_types_by_name['BatchingOptions']
_OPTIMIZEROPTIONS_LEVEL = _OPTIMIZEROPTIONS.enum_types_by_name['Level']
_OPTIMIZEROPTIONS_GLOBALJITLEVEL = _OPTIMIZEROPTIONS.enum_types_by_name['GlobalJitLevel']
_CONFIGPROTO_EXPERIMENTAL_MLIRBRIDGEROLLOUT = _CONFIGPROTO_EXPERIMENTAL.enum_types_by_name['MlirBridgeRollout']
_RUNOPTIONS_TRACELEVEL = _RUNOPTIONS.enum_types_by_name['TraceLevel']
GPUOptions = _reflection.GeneratedProtocolMessageType('GPUOptions', (_message.Message,), {
'Experimental' : _reflection.GeneratedProtocolMessageType('Experimental', (_message.Message,), {
'VirtualDevices' : _reflection.GeneratedProtocolMessageType('VirtualDevices', (_message.Message,), {
'DESCRIPTOR' : _GPUOPTIONS_EXPERIMENTAL_VIRTUALDEVICES,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GPUOptions.Experimental.VirtualDevices)
})
,
'StreamMergeOptions' : _reflection.GeneratedProtocolMessageType('StreamMergeOptions', (_message.Message,), {
'DESCRIPTOR' : _GPUOPTIONS_EXPERIMENTAL_STREAMMERGEOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GPUOptions.Experimental.StreamMergeOptions)
})
,
'DESCRIPTOR' : _GPUOPTIONS_EXPERIMENTAL,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GPUOptions.Experimental)
})
,
'DESCRIPTOR' : _GPUOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GPUOptions)
})
_sym_db.RegisterMessage(GPUOptions)
_sym_db.RegisterMessage(GPUOptions.Experimental)
_sym_db.RegisterMessage(GPUOptions.Experimental.VirtualDevices)
_sym_db.RegisterMessage(GPUOptions.Experimental.StreamMergeOptions)
OptimizerOptions = _reflection.GeneratedProtocolMessageType('OptimizerOptions', (_message.Message,), {
'DESCRIPTOR' : _OPTIMIZEROPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OptimizerOptions)
})
_sym_db.RegisterMessage(OptimizerOptions)
GraphOptions = _reflection.GeneratedProtocolMessageType('GraphOptions', (_message.Message,), {
'DESCRIPTOR' : _GRAPHOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphOptions)
})
_sym_db.RegisterMessage(GraphOptions)
ThreadPoolOptionProto = _reflection.GeneratedProtocolMessageType('ThreadPoolOptionProto', (_message.Message,), {
'DESCRIPTOR' : _THREADPOOLOPTIONPROTO,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ThreadPoolOptionProto)
})
_sym_db.RegisterMessage(ThreadPoolOptionProto)
SessionMetadata = _reflection.GeneratedProtocolMessageType('SessionMetadata', (_message.Message,), {
'DESCRIPTOR' : _SESSIONMETADATA,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SessionMetadata)
})
_sym_db.RegisterMessage(SessionMetadata)
ConfigProto = _reflection.GeneratedProtocolMessageType('ConfigProto', (_message.Message,), {
'DeviceCountEntry' : _reflection.GeneratedProtocolMessageType('DeviceCountEntry', (_message.Message,), {
'DESCRIPTOR' : _CONFIGPROTO_DEVICECOUNTENTRY,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ConfigProto.DeviceCountEntry)
})
,
'Experimental' : _reflection.GeneratedProtocolMessageType('Experimental', (_message.Message,), {
'DESCRIPTOR' : _CONFIGPROTO_EXPERIMENTAL,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ConfigProto.Experimental)
})
,
'DESCRIPTOR' : _CONFIGPROTO,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ConfigProto)
})
_sym_db.RegisterMessage(ConfigProto)
_sym_db.RegisterMessage(ConfigProto.DeviceCountEntry)
_sym_db.RegisterMessage(ConfigProto.Experimental)
RunOptions = _reflection.GeneratedProtocolMessageType('RunOptions', (_message.Message,), {
'Experimental' : _reflection.GeneratedProtocolMessageType('Experimental', (_message.Message,), {
'RunHandlerPoolOptions' : _reflection.GeneratedProtocolMessageType('RunHandlerPoolOptions', (_message.Message,), {
'DESCRIPTOR' : _RUNOPTIONS_EXPERIMENTAL_RUNHANDLERPOOLOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RunOptions.Experimental.RunHandlerPoolOptions)
})
,
'DESCRIPTOR' : _RUNOPTIONS_EXPERIMENTAL,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RunOptions.Experimental)
})
,
'DESCRIPTOR' : _RUNOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RunOptions)
})
_sym_db.RegisterMessage(RunOptions)
_sym_db.RegisterMessage(RunOptions.Experimental)
_sym_db.RegisterMessage(RunOptions.Experimental.RunHandlerPoolOptions)
RunMetadata = _reflection.GeneratedProtocolMessageType('RunMetadata', (_message.Message,), {
'FunctionGraphs' : _reflection.GeneratedProtocolMessageType('FunctionGraphs', (_message.Message,), {
'DESCRIPTOR' : _RUNMETADATA_FUNCTIONGRAPHS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RunMetadata.FunctionGraphs)
})
,
'DESCRIPTOR' : _RUNMETADATA,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RunMetadata)
})
_sym_db.RegisterMessage(RunMetadata)
_sym_db.RegisterMessage(RunMetadata.FunctionGraphs)
TensorConnection = _reflection.GeneratedProtocolMessageType('TensorConnection', (_message.Message,), {
'DESCRIPTOR' : _TENSORCONNECTION,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.TensorConnection)
})
_sym_db.RegisterMessage(TensorConnection)
CallableOptions = _reflection.GeneratedProtocolMessageType('CallableOptions', (_message.Message,), {
'FeedDevicesEntry' : _reflection.GeneratedProtocolMessageType('FeedDevicesEntry', (_message.Message,), {
'DESCRIPTOR' : _CALLABLEOPTIONS_FEEDDEVICESENTRY,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CallableOptions.FeedDevicesEntry)
})
,
'FetchDevicesEntry' : _reflection.GeneratedProtocolMessageType('FetchDevicesEntry', (_message.Message,), {
'DESCRIPTOR' : _CALLABLEOPTIONS_FETCHDEVICESENTRY,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CallableOptions.FetchDevicesEntry)
})
,
'DESCRIPTOR' : _CALLABLEOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CallableOptions)
})
_sym_db.RegisterMessage(CallableOptions)
_sym_db.RegisterMessage(CallableOptions.FeedDevicesEntry)
_sym_db.RegisterMessage(CallableOptions.FetchDevicesEntry)
BatchingOptions = _reflection.GeneratedProtocolMessageType('BatchingOptions', (_message.Message,), {
'DESCRIPTOR' : _BATCHINGOPTIONS,
'__module__' : 'tensorboard.compat.proto.config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.BatchingOptions)
})
_sym_db.RegisterMessage(BatchingOptions)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\014ConfigProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_CONFIGPROTO_DEVICECOUNTENTRY._options = None
_CONFIGPROTO_DEVICECOUNTENTRY._serialized_options = b'8\001'
_CALLABLEOPTIONS_FEEDDEVICESENTRY._options = None
_CALLABLEOPTIONS_FEEDDEVICESENTRY._serialized_options = b'8\001'
_CALLABLEOPTIONS_FETCHDEVICESENTRY._options = None
_CALLABLEOPTIONS_FETCHDEVICESENTRY._serialized_options = b'8\001'
_GPUOPTIONS._serialized_start=401
_GPUOPTIONS._serialized_end=1693
_GPUOPTIONS_EXPERIMENTAL._serialized_start=729
_GPUOPTIONS_EXPERIMENTAL._serialized_end=1693
_GPUOPTIONS_EXPERIMENTAL_VIRTUALDEVICES._serialized_start=1474
_GPUOPTIONS_EXPERIMENTAL_VIRTUALDEVICES._serialized_end=1557
_GPUOPTIONS_EXPERIMENTAL_STREAMMERGEOPTIONS._serialized_start=1560
_GPUOPTIONS_EXPERIMENTAL_STREAMMERGEOPTIONS._serialized_end=1693
_OPTIMIZEROPTIONS._serialized_start=1696
_OPTIMIZEROPTIONS._serialized_end=2111
_OPTIMIZEROPTIONS_LEVEL._serialized_start=2010
_OPTIMIZEROPTIONS_LEVEL._serialized_end=2042
_OPTIMIZEROPTIONS_GLOBALJITLEVEL._serialized_start=2044
_OPTIMIZEROPTIONS_GLOBALJITLEVEL._serialized_end=2111
_GRAPHOPTIONS._serialized_start=2114
_GRAPHOPTIONS._serialized_end=2482
_THREADPOOLOPTIONPROTO._serialized_start=2484
_THREADPOOLOPTIONPROTO._serialized_end=2549
_SESSIONMETADATA._serialized_start=2551
_SESSIONMETADATA._serialized_end=2599
_CONFIGPROTO._serialized_start=2602
_CONFIGPROTO._serialized_end=4841
_CONFIGPROTO_DEVICECOUNTENTRY._serialized_start=3389
_CONFIGPROTO_DEVICECOUNTENTRY._serialized_end=3439
_CONFIGPROTO_EXPERIMENTAL._serialized_start=3442
_CONFIGPROTO_EXPERIMENTAL._serialized_end=4841
_CONFIGPROTO_EXPERIMENTAL_MLIRBRIDGEROLLOUT._serialized_start=4595
_CONFIGPROTO_EXPERIMENTAL_MLIRBRIDGEROLLOUT._serialized_end=4817
_RUNOPTIONS._serialized_start=4844
_RUNOPTIONS._serialized_end=5457
_RUNOPTIONS_EXPERIMENTAL._serialized_start=5156
_RUNOPTIONS_EXPERIMENTAL._serialized_end=5367
_RUNOPTIONS_EXPERIMENTAL_RUNHANDLERPOOLOPTIONS._serialized_start=5326
_RUNOPTIONS_EXPERIMENTAL_RUNHANDLERPOOLOPTIONS._serialized_end=5367
_RUNOPTIONS_TRACELEVEL._serialized_start=5369
_RUNOPTIONS_TRACELEVEL._serialized_end=5451
_RUNMETADATA._serialized_start=5460
_RUNMETADATA._serialized_end=5914
_RUNMETADATA_FUNCTIONGRAPHS._serialized_start=5738
_RUNMETADATA_FUNCTIONGRAPHS._serialized_end=5914
_TENSORCONNECTION._serialized_start=5916
_TENSORCONNECTION._serialized_end=5974
_CALLABLEOPTIONS._serialized_start=5977
_CALLABLEOPTIONS._serialized_end=6413
_CALLABLEOPTIONS_FEEDDEVICESENTRY._serialized_start=6310
_CALLABLEOPTIONS_FEEDDEVICESENTRY._serialized_end=6360
_CALLABLEOPTIONS_FETCHDEVICESENTRY._serialized_start=6362
_CALLABLEOPTIONS_FETCHDEVICESENTRY._serialized_end=6413
_BATCHINGOPTIONS._serialized_start=6416
_BATCHINGOPTIONS._serialized_end=6573
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/coordination_config.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2tensorboard/compat/proto/coordination_config.proto\x12\x0btensorboard\"1\n\x0e\x43oordinatedJob\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tnum_tasks\x18\x02 \x01(\x05\"\xf8\x03\n\x19\x43oordinationServiceConfig\x12\x14\n\x0cservice_type\x18\x01 \x01(\t\x12\x16\n\x0eservice_leader\x18\x02 \x01(\t\x12\x1b\n\x13\x65nable_health_check\x18\x03 \x01(\x08\x12&\n\x1e\x63luster_register_timeout_in_ms\x18\x04 \x01(\x03\x12%\n\x1d\x63luster_register_with_barrier\x18\x0e \x01(\x08\x12\x1f\n\x17heartbeat_timeout_in_ms\x18\x05 \x01(\x03\x12\x39\n\x14\x63oordinated_job_list\x18\n \x03(\x0b\x32\x1b.tensorboard.CoordinatedJob\x12&\n\x1eshutdown_barrier_timeout_in_ms\x18\x07 \x01(\x03\x12*\n\"agent_destruction_without_shutdown\x18\x08 \x01(\x08\x12\x18\n\x10recoverable_jobs\x18\t \x03(\t\x12*\n\"allow_new_incarnation_to_reconnect\x18\x0b \x01(\x08\x12\x15\n\rforce_disable\x18\x0c \x01(\x08\x12.\n&poll_for_error_from_service_at_startup\x18\r \x01(\x08J\x04\x08\x06\x10\x07\x42WZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_protob\x06proto3')
_COORDINATEDJOB = DESCRIPTOR.message_types_by_name['CoordinatedJob']
_COORDINATIONSERVICECONFIG = DESCRIPTOR.message_types_by_name['CoordinationServiceConfig']
CoordinatedJob = _reflection.GeneratedProtocolMessageType('CoordinatedJob', (_message.Message,), {
'DESCRIPTOR' : _COORDINATEDJOB,
'__module__' : 'tensorboard.compat.proto.coordination_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CoordinatedJob)
})
_sym_db.RegisterMessage(CoordinatedJob)
CoordinationServiceConfig = _reflection.GeneratedProtocolMessageType('CoordinationServiceConfig', (_message.Message,), {
'DESCRIPTOR' : _COORDINATIONSERVICECONFIG,
'__module__' : 'tensorboard.compat.proto.coordination_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CoordinationServiceConfig)
})
_sym_db.RegisterMessage(CoordinationServiceConfig)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto'
_COORDINATEDJOB._serialized_start=67
_COORDINATEDJOB._serialized_end=116
_COORDINATIONSERVICECONFIG._serialized_start=119
_COORDINATIONSERVICECONFIG._serialized_end=623
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,87 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/cost_graph.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import tensor_shape_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)tensorboard/compat/proto/cost_graph.proto\x12\x0btensorboard\x1a+tensorboard/compat/proto/tensor_shape.proto\x1a$tensorboard/compat/proto/types.proto\"\xd0\x06\n\x0c\x43ostGraphDef\x12,\n\x04node\x18\x01 \x03(\x0b\x32\x1e.tensorboard.CostGraphDef.Node\x12\x36\n\x04\x63ost\x18\x02 \x03(\x0b\x32(.tensorboard.CostGraphDef.AggregatedCost\x1a\xa6\x05\n\x04Node\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x64\x65vice\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\x05\x12<\n\ninput_info\x18\x04 \x03(\x0b\x32(.tensorboard.CostGraphDef.Node.InputInfo\x12>\n\x0boutput_info\x18\x05 \x03(\x0b\x32).tensorboard.CostGraphDef.Node.OutputInfo\x12\x1d\n\x15temporary_memory_size\x18\x06 \x01(\x03\x12\x1e\n\x16persistent_memory_size\x18\x0c \x01(\x03\x12!\n\x15host_temp_memory_size\x18\n \x01(\x03\x42\x02\x18\x01\x12#\n\x17\x64\x65vice_temp_memory_size\x18\x0b \x01(\x03\x42\x02\x18\x01\x12)\n\x1d\x64\x65vice_persistent_memory_size\x18\x10 \x01(\x03\x42\x02\x18\x01\x12\x14\n\x0c\x63ompute_cost\x18\t \x01(\x03\x12\x14\n\x0c\x63ompute_time\x18\x0e \x01(\x03\x12\x13\n\x0bmemory_time\x18\x0f \x01(\x03\x12\x10\n\x08is_final\x18\x07 \x01(\x08\x12\x15\n\rcontrol_input\x18\x08 \x03(\x05\x12\x12\n\ninaccurate\x18\x11 \x01(\x08\x1a;\n\tInputInfo\x12\x16\n\x0epreceding_node\x18\x01 \x01(\x05\x12\x16\n\x0epreceding_port\x18\x02 \x01(\x05\x1a\x88\x01\n\nOutputInfo\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x18\n\x10\x61lias_input_port\x18\x02 \x01(\x03\x12,\n\x05shape\x18\x03 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProto\x12$\n\x05\x64type\x18\x04 \x01(\x0e\x32\x15.tensorboard.DataType\x1a\x31\n\x0e\x41ggregatedCost\x12\x0c\n\x04\x63ost\x18\x01 \x01(\x02\x12\x11\n\tdimension\x18\x02 \x01(\tB\x83\x01\n\x18org.tensorflow.frameworkB\x0f\x43ostGraphProtosP\x01ZQgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/cost_graph_go_proto\xf8\x01\x01\x62\x06proto3')
_COSTGRAPHDEF = DESCRIPTOR.message_types_by_name['CostGraphDef']
_COSTGRAPHDEF_NODE = _COSTGRAPHDEF.nested_types_by_name['Node']
_COSTGRAPHDEF_NODE_INPUTINFO = _COSTGRAPHDEF_NODE.nested_types_by_name['InputInfo']
_COSTGRAPHDEF_NODE_OUTPUTINFO = _COSTGRAPHDEF_NODE.nested_types_by_name['OutputInfo']
_COSTGRAPHDEF_AGGREGATEDCOST = _COSTGRAPHDEF.nested_types_by_name['AggregatedCost']
CostGraphDef = _reflection.GeneratedProtocolMessageType('CostGraphDef', (_message.Message,), {
'Node' : _reflection.GeneratedProtocolMessageType('Node', (_message.Message,), {
'InputInfo' : _reflection.GeneratedProtocolMessageType('InputInfo', (_message.Message,), {
'DESCRIPTOR' : _COSTGRAPHDEF_NODE_INPUTINFO,
'__module__' : 'tensorboard.compat.proto.cost_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CostGraphDef.Node.InputInfo)
})
,
'OutputInfo' : _reflection.GeneratedProtocolMessageType('OutputInfo', (_message.Message,), {
'DESCRIPTOR' : _COSTGRAPHDEF_NODE_OUTPUTINFO,
'__module__' : 'tensorboard.compat.proto.cost_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CostGraphDef.Node.OutputInfo)
})
,
'DESCRIPTOR' : _COSTGRAPHDEF_NODE,
'__module__' : 'tensorboard.compat.proto.cost_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CostGraphDef.Node)
})
,
'AggregatedCost' : _reflection.GeneratedProtocolMessageType('AggregatedCost', (_message.Message,), {
'DESCRIPTOR' : _COSTGRAPHDEF_AGGREGATEDCOST,
'__module__' : 'tensorboard.compat.proto.cost_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CostGraphDef.AggregatedCost)
})
,
'DESCRIPTOR' : _COSTGRAPHDEF,
'__module__' : 'tensorboard.compat.proto.cost_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CostGraphDef)
})
_sym_db.RegisterMessage(CostGraphDef)
_sym_db.RegisterMessage(CostGraphDef.Node)
_sym_db.RegisterMessage(CostGraphDef.Node.InputInfo)
_sym_db.RegisterMessage(CostGraphDef.Node.OutputInfo)
_sym_db.RegisterMessage(CostGraphDef.AggregatedCost)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\017CostGraphProtosP\001ZQgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/cost_graph_go_proto\370\001\001'
_COSTGRAPHDEF_NODE.fields_by_name['host_temp_memory_size']._options = None
_COSTGRAPHDEF_NODE.fields_by_name['host_temp_memory_size']._serialized_options = b'\030\001'
_COSTGRAPHDEF_NODE.fields_by_name['device_temp_memory_size']._options = None
_COSTGRAPHDEF_NODE.fields_by_name['device_temp_memory_size']._serialized_options = b'\030\001'
_COSTGRAPHDEF_NODE.fields_by_name['device_persistent_memory_size']._options = None
_COSTGRAPHDEF_NODE.fields_by_name['device_persistent_memory_size']._serialized_options = b'\030\001'
_COSTGRAPHDEF._serialized_start=142
_COSTGRAPHDEF._serialized_end=990
_COSTGRAPHDEF_NODE._serialized_start=261
_COSTGRAPHDEF_NODE._serialized_end=939
_COSTGRAPHDEF_NODE_INPUTINFO._serialized_start=741
_COSTGRAPHDEF_NODE_INPUTINFO._serialized_end=800
_COSTGRAPHDEF_NODE_OUTPUTINFO._serialized_start=803
_COSTGRAPHDEF_NODE_OUTPUTINFO._serialized_end=939
_COSTGRAPHDEF_AGGREGATEDCOST._serialized_start=941
_COSTGRAPHDEF_AGGREGATEDCOST._serialized_end=990
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/cpp_shape_inference.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import full_type_pb2 as tensorboard_dot_compat_dot_proto_dot_full__type__pb2
from tensorboard.compat.proto import tensor_shape_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2tensorboard/compat/proto/cpp_shape_inference.proto\x12\x0btensorboard\x1a(tensorboard/compat/proto/full_type.proto\x1a+tensorboard/compat/proto/tensor_shape.proto\x1a$tensorboard/compat/proto/types.proto\"\xa1\x03\n\x17\x43ppShapeInferenceResult\x12,\n\x05shape\x18\x01 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProto\x12\x44\n\x0bhandle_data\x18\x04 \x01(\x0b\x32/.tensorboard.CppShapeInferenceResult.HandleData\x1a\x96\x01\n\x12HandleShapeAndType\x12,\n\x05shape\x18\x01 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProto\x12$\n\x05\x64type\x18\x02 \x01(\x0e\x32\x15.tensorboard.DataType\x12&\n\x04type\x18\x04 \x01(\x0b\x32\x18.tensorboard.FullTypeDefJ\x04\x08\x03\x10\x04\x1am\n\nHandleData\x12\x0e\n\x06is_set\x18\x01 \x01(\x08\x12O\n\x0eshape_and_type\x18\x02 \x03(\x0b\x32\x37.tensorboard.CppShapeInferenceResult.HandleShapeAndTypeJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"e\n\x1d\x43ppShapeInferenceInputsNeeded\x12\x1c\n\x14input_tensors_needed\x18\x01 \x03(\x05\x12&\n\x1einput_tensors_as_shapes_needed\x18\x02 \x03(\x05\x42\x61Z\\github.com/tensorflow/tensorflow/tensorflow/go/python/framework/cpp_shape_inference_go_proto\xf8\x01\x01\x62\x06proto3')
_CPPSHAPEINFERENCERESULT = DESCRIPTOR.message_types_by_name['CppShapeInferenceResult']
_CPPSHAPEINFERENCERESULT_HANDLESHAPEANDTYPE = _CPPSHAPEINFERENCERESULT.nested_types_by_name['HandleShapeAndType']
_CPPSHAPEINFERENCERESULT_HANDLEDATA = _CPPSHAPEINFERENCERESULT.nested_types_by_name['HandleData']
_CPPSHAPEINFERENCEINPUTSNEEDED = DESCRIPTOR.message_types_by_name['CppShapeInferenceInputsNeeded']
CppShapeInferenceResult = _reflection.GeneratedProtocolMessageType('CppShapeInferenceResult', (_message.Message,), {
'HandleShapeAndType' : _reflection.GeneratedProtocolMessageType('HandleShapeAndType', (_message.Message,), {
'DESCRIPTOR' : _CPPSHAPEINFERENCERESULT_HANDLESHAPEANDTYPE,
'__module__' : 'tensorboard.compat.proto.cpp_shape_inference_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CppShapeInferenceResult.HandleShapeAndType)
})
,
'HandleData' : _reflection.GeneratedProtocolMessageType('HandleData', (_message.Message,), {
'DESCRIPTOR' : _CPPSHAPEINFERENCERESULT_HANDLEDATA,
'__module__' : 'tensorboard.compat.proto.cpp_shape_inference_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CppShapeInferenceResult.HandleData)
})
,
'DESCRIPTOR' : _CPPSHAPEINFERENCERESULT,
'__module__' : 'tensorboard.compat.proto.cpp_shape_inference_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CppShapeInferenceResult)
})
_sym_db.RegisterMessage(CppShapeInferenceResult)
_sym_db.RegisterMessage(CppShapeInferenceResult.HandleShapeAndType)
_sym_db.RegisterMessage(CppShapeInferenceResult.HandleData)
CppShapeInferenceInputsNeeded = _reflection.GeneratedProtocolMessageType('CppShapeInferenceInputsNeeded', (_message.Message,), {
'DESCRIPTOR' : _CPPSHAPEINFERENCEINPUTSNEEDED,
'__module__' : 'tensorboard.compat.proto.cpp_shape_inference_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CppShapeInferenceInputsNeeded)
})
_sym_db.RegisterMessage(CppShapeInferenceInputsNeeded)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'Z\\github.com/tensorflow/tensorflow/tensorflow/go/python/framework/cpp_shape_inference_go_proto\370\001\001'
_CPPSHAPEINFERENCERESULT._serialized_start=193
_CPPSHAPEINFERENCERESULT._serialized_end=610
_CPPSHAPEINFERENCERESULT_HANDLESHAPEANDTYPE._serialized_start=337
_CPPSHAPEINFERENCERESULT_HANDLESHAPEANDTYPE._serialized_end=487
_CPPSHAPEINFERENCERESULT_HANDLEDATA._serialized_start=489
_CPPSHAPEINFERENCERESULT_HANDLEDATA._serialized_end=598
_CPPSHAPEINFERENCEINPUTSNEEDED._serialized_start=612
_CPPSHAPEINFERENCEINPUTSNEEDED._serialized_end=713
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/debug.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$tensorboard/compat/proto/debug.proto\x12\x0btensorboard\"\x8e\x01\n\x10\x44\x65\x62ugTensorWatch\x12\x11\n\tnode_name\x18\x01 \x01(\t\x12\x13\n\x0boutput_slot\x18\x02 \x01(\x05\x12\x11\n\tdebug_ops\x18\x03 \x03(\t\x12\x12\n\ndebug_urls\x18\x04 \x03(\t\x12+\n#tolerate_debug_op_creation_failures\x18\x05 \x01(\x08\"\x82\x01\n\x0c\x44\x65\x62ugOptions\x12>\n\x17\x64\x65\x62ug_tensor_watch_opts\x18\x04 \x03(\x0b\x32\x1d.tensorboard.DebugTensorWatch\x12\x13\n\x0bglobal_step\x18\n \x01(\x03\x12\x1d\n\x15reset_disk_byte_usage\x18\x0b \x01(\x08\"j\n\x12\x44\x65\x62uggedSourceFile\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x11\n\tfile_path\x18\x02 \x01(\t\x12\x15\n\rlast_modified\x18\x03 \x01(\x03\x12\r\n\x05\x62ytes\x18\x04 \x01(\x03\x12\r\n\x05lines\x18\x05 \x03(\t\"L\n\x13\x44\x65\x62uggedSourceFiles\x12\x35\n\x0csource_files\x18\x01 \x03(\x0b\x32\x1f.tensorboard.DebuggedSourceFileB\x83\x01\n\x18org.tensorflow.frameworkB\x0b\x44\x65\x62ugProtosP\x01ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\xf8\x01\x01\x62\x06proto3')
_DEBUGTENSORWATCH = DESCRIPTOR.message_types_by_name['DebugTensorWatch']
_DEBUGOPTIONS = DESCRIPTOR.message_types_by_name['DebugOptions']
_DEBUGGEDSOURCEFILE = DESCRIPTOR.message_types_by_name['DebuggedSourceFile']
_DEBUGGEDSOURCEFILES = DESCRIPTOR.message_types_by_name['DebuggedSourceFiles']
DebugTensorWatch = _reflection.GeneratedProtocolMessageType('DebugTensorWatch', (_message.Message,), {
'DESCRIPTOR' : _DEBUGTENSORWATCH,
'__module__' : 'tensorboard.compat.proto.debug_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.DebugTensorWatch)
})
_sym_db.RegisterMessage(DebugTensorWatch)
DebugOptions = _reflection.GeneratedProtocolMessageType('DebugOptions', (_message.Message,), {
'DESCRIPTOR' : _DEBUGOPTIONS,
'__module__' : 'tensorboard.compat.proto.debug_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.DebugOptions)
})
_sym_db.RegisterMessage(DebugOptions)
DebuggedSourceFile = _reflection.GeneratedProtocolMessageType('DebuggedSourceFile', (_message.Message,), {
'DESCRIPTOR' : _DEBUGGEDSOURCEFILE,
'__module__' : 'tensorboard.compat.proto.debug_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.DebuggedSourceFile)
})
_sym_db.RegisterMessage(DebuggedSourceFile)
DebuggedSourceFiles = _reflection.GeneratedProtocolMessageType('DebuggedSourceFiles', (_message.Message,), {
'DESCRIPTOR' : _DEBUGGEDSOURCEFILES,
'__module__' : 'tensorboard.compat.proto.debug_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.DebuggedSourceFiles)
})
_sym_db.RegisterMessage(DebuggedSourceFiles)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\013DebugProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_DEBUGTENSORWATCH._serialized_start=54
_DEBUGTENSORWATCH._serialized_end=196
_DEBUGOPTIONS._serialized_start=199
_DEBUGOPTIONS._serialized_end=329
_DEBUGGEDSOURCEFILE._serialized_start=331
_DEBUGGEDSOURCEFILE._serialized_end=437
_DEBUGGEDSOURCEFILES._serialized_start=439
_DEBUGGEDSOURCEFILES._serialized_end=515
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,149 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/event.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import enum_type_wrapper
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import summary_pb2 as tensorboard_dot_compat_dot_proto_dot_summary__pb2
try:
tensorboard_dot_compat_dot_proto_dot_histogram__pb2 = tensorboard_dot_compat_dot_proto_dot_summary__pb2.tensorboard_dot_compat_dot_proto_dot_histogram__pb2
except AttributeError:
tensorboard_dot_compat_dot_proto_dot_histogram__pb2 = tensorboard_dot_compat_dot_proto_dot_summary__pb2.tensorboard.compat.proto.histogram_pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$tensorboard/compat/proto/event.proto\x12\x0btensorboard\x1a&tensorboard/compat/proto/summary.proto\"\xf9\x02\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x01(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x16\n\x0c\x66ile_version\x18\x03 \x01(\tH\x00\x12\x13\n\tgraph_def\x18\x04 \x01(\x0cH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.tensorboard.SummaryH\x00\x12\x32\n\x0blog_message\x18\x06 \x01(\x0b\x32\x17.tensorboard.LogMessageB\x02\x18\x01H\x00\x12.\n\x0bsession_log\x18\x07 \x01(\x0b\x32\x17.tensorboard.SessionLogH\x00\x12=\n\x13tagged_run_metadata\x18\x08 \x01(\x0b\x32\x1e.tensorboard.TaggedRunMetadataH\x00\x12\x18\n\x0emeta_graph_def\x18\t \x01(\x0cH\x00\x12\x34\n\x0fsource_metadata\x18\n \x01(\x0b\x32\x1b.tensorboard.SourceMetadataB\x06\n\x04what\" \n\x0eSourceMetadata\x12\x0e\n\x06writer\x18\x01 \x01(\t\"\xa2\x01\n\nLogMessage\x12,\n\x05level\x18\x01 \x01(\x0e\x32\x1d.tensorboard.LogMessage.Level\x12\x0f\n\x07message\x18\x02 \x01(\t\"Q\n\x05Level\x12\x0b\n\x07UNKNOWN\x10\x00\x12\r\n\tDEBUGGING\x10\n\x12\x08\n\x04INFO\x10\x14\x12\x08\n\x04WARN\x10\x1e\x12\t\n\x05\x45RROR\x10(\x12\t\n\x05\x46\x41TAL\x10\x32\x1a\x02\x18\x01:\x02\x18\x01\"\xb7\x01\n\nSessionLog\x12\x35\n\x06status\x18\x01 \x01(\x0e\x32%.tensorboard.SessionLog.SessionStatus\x12\x17\n\x0f\x63heckpoint_path\x18\x02 \x01(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t\"L\n\rSessionStatus\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\t\n\x05START\x10\x01\x12\x08\n\x04STOP\x10\x02\x12\x0e\n\nCHECKPOINT\x10\x03\"6\n\x11TaggedRunMetadata\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\x14\n\x0crun_metadata\x18\x02 \x01(\x0c\"$\n\x0eWatchdogConfig\x12\x12\n\ntimeout_ms\x18\x01 \x01(\x03\"&\n\x11RequestedExitCode\x12\x11\n\texit_code\x18\x01 \x01(\x05\"\xb9\x01\n\x16WorkerHeartbeatRequest\x12\x36\n\rshutdown_mode\x18\x01 \x01(\x0e\x32\x1f.tensorboard.WorkerShutdownMode\x12\x34\n\x0fwatchdog_config\x18\x02 \x01(\x0b\x32\x1b.tensorboard.WatchdogConfig\x12\x31\n\texit_code\x18\x03 \x01(\x0b\x32\x1e.tensorboard.RequestedExitCode\"\x85\x01\n\x17WorkerHeartbeatResponse\x12\x30\n\rhealth_status\x18\x01 \x01(\x0e\x32\x19.tensorboard.WorkerHealth\x12&\n\nworker_log\x18\x02 \x03(\x0b\x32\x12.tensorboard.Event\x12\x10\n\x08hostname\x18\x03 \x01(\t*[\n\x0cWorkerHealth\x12\x06\n\x02OK\x10\x00\x12\x1c\n\x18RECEIVED_SHUTDOWN_SIGNAL\x10\x01\x12\x12\n\x0eINTERNAL_ERROR\x10\x02\x12\x11\n\rSHUTTING_DOWN\x10\x03*k\n\x12WorkerShutdownMode\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x12\n\x0eNOT_CONFIGURED\x10\x01\x12\x18\n\x14WAIT_FOR_COORDINATOR\x10\x02\x12\x1a\n\x16SHUTDOWN_AFTER_TIMEOUT\x10\x03\x42p\n\x13org.tensorflow.utilB\x0b\x45ventProtosP\x01ZGgithub.com/tensorflow/tensorflow/tensorflow/go/core/util/event_go_proto\xf8\x01\x01\x62\x06proto3')
_WORKERHEALTH = DESCRIPTOR.enum_types_by_name['WorkerHealth']
WorkerHealth = enum_type_wrapper.EnumTypeWrapper(_WORKERHEALTH)
_WORKERSHUTDOWNMODE = DESCRIPTOR.enum_types_by_name['WorkerShutdownMode']
WorkerShutdownMode = enum_type_wrapper.EnumTypeWrapper(_WORKERSHUTDOWNMODE)
OK = 0
RECEIVED_SHUTDOWN_SIGNAL = 1
INTERNAL_ERROR = 2
SHUTTING_DOWN = 3
DEFAULT = 0
NOT_CONFIGURED = 1
WAIT_FOR_COORDINATOR = 2
SHUTDOWN_AFTER_TIMEOUT = 3
_EVENT = DESCRIPTOR.message_types_by_name['Event']
_SOURCEMETADATA = DESCRIPTOR.message_types_by_name['SourceMetadata']
_LOGMESSAGE = DESCRIPTOR.message_types_by_name['LogMessage']
_SESSIONLOG = DESCRIPTOR.message_types_by_name['SessionLog']
_TAGGEDRUNMETADATA = DESCRIPTOR.message_types_by_name['TaggedRunMetadata']
_WATCHDOGCONFIG = DESCRIPTOR.message_types_by_name['WatchdogConfig']
_REQUESTEDEXITCODE = DESCRIPTOR.message_types_by_name['RequestedExitCode']
_WORKERHEARTBEATREQUEST = DESCRIPTOR.message_types_by_name['WorkerHeartbeatRequest']
_WORKERHEARTBEATRESPONSE = DESCRIPTOR.message_types_by_name['WorkerHeartbeatResponse']
_LOGMESSAGE_LEVEL = _LOGMESSAGE.enum_types_by_name['Level']
_SESSIONLOG_SESSIONSTATUS = _SESSIONLOG.enum_types_by_name['SessionStatus']
Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), {
'DESCRIPTOR' : _EVENT,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.Event)
})
_sym_db.RegisterMessage(Event)
SourceMetadata = _reflection.GeneratedProtocolMessageType('SourceMetadata', (_message.Message,), {
'DESCRIPTOR' : _SOURCEMETADATA,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SourceMetadata)
})
_sym_db.RegisterMessage(SourceMetadata)
LogMessage = _reflection.GeneratedProtocolMessageType('LogMessage', (_message.Message,), {
'DESCRIPTOR' : _LOGMESSAGE,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.LogMessage)
})
_sym_db.RegisterMessage(LogMessage)
SessionLog = _reflection.GeneratedProtocolMessageType('SessionLog', (_message.Message,), {
'DESCRIPTOR' : _SESSIONLOG,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SessionLog)
})
_sym_db.RegisterMessage(SessionLog)
TaggedRunMetadata = _reflection.GeneratedProtocolMessageType('TaggedRunMetadata', (_message.Message,), {
'DESCRIPTOR' : _TAGGEDRUNMETADATA,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.TaggedRunMetadata)
})
_sym_db.RegisterMessage(TaggedRunMetadata)
WatchdogConfig = _reflection.GeneratedProtocolMessageType('WatchdogConfig', (_message.Message,), {
'DESCRIPTOR' : _WATCHDOGCONFIG,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.WatchdogConfig)
})
_sym_db.RegisterMessage(WatchdogConfig)
RequestedExitCode = _reflection.GeneratedProtocolMessageType('RequestedExitCode', (_message.Message,), {
'DESCRIPTOR' : _REQUESTEDEXITCODE,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RequestedExitCode)
})
_sym_db.RegisterMessage(RequestedExitCode)
WorkerHeartbeatRequest = _reflection.GeneratedProtocolMessageType('WorkerHeartbeatRequest', (_message.Message,), {
'DESCRIPTOR' : _WORKERHEARTBEATREQUEST,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.WorkerHeartbeatRequest)
})
_sym_db.RegisterMessage(WorkerHeartbeatRequest)
WorkerHeartbeatResponse = _reflection.GeneratedProtocolMessageType('WorkerHeartbeatResponse', (_message.Message,), {
'DESCRIPTOR' : _WORKERHEARTBEATRESPONSE,
'__module__' : 'tensorboard.compat.proto.event_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.WorkerHeartbeatResponse)
})
_sym_db.RegisterMessage(WorkerHeartbeatResponse)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\023org.tensorflow.utilB\013EventProtosP\001ZGgithub.com/tensorflow/tensorflow/tensorflow/go/core/util/event_go_proto\370\001\001'
_EVENT.fields_by_name['log_message']._options = None
_EVENT.fields_by_name['log_message']._serialized_options = b'\030\001'
_LOGMESSAGE_LEVEL._options = None
_LOGMESSAGE_LEVEL._serialized_options = b'\030\001'
_LOGMESSAGE._options = None
_LOGMESSAGE._serialized_options = b'\030\001'
_WORKERHEALTH._serialized_start=1316
_WORKERHEALTH._serialized_end=1407
_WORKERSHUTDOWNMODE._serialized_start=1409
_WORKERSHUTDOWNMODE._serialized_end=1516
_EVENT._serialized_start=94
_EVENT._serialized_end=471
_SOURCEMETADATA._serialized_start=473
_SOURCEMETADATA._serialized_end=505
_LOGMESSAGE._serialized_start=508
_LOGMESSAGE._serialized_end=670
_LOGMESSAGE_LEVEL._serialized_start=585
_LOGMESSAGE_LEVEL._serialized_end=666
_SESSIONLOG._serialized_start=673
_SESSIONLOG._serialized_end=856
_SESSIONLOG_SESSIONSTATUS._serialized_start=780
_SESSIONLOG_SESSIONSTATUS._serialized_end=856
_TAGGEDRUNMETADATA._serialized_start=858
_TAGGEDRUNMETADATA._serialized_end=912
_WATCHDOGCONFIG._serialized_start=914
_WATCHDOGCONFIG._serialized_end=950
_REQUESTEDEXITCODE._serialized_start=952
_REQUESTEDEXITCODE._serialized_end=990
_WORKERHEARTBEATREQUEST._serialized_start=993
_WORKERHEARTBEATREQUEST._serialized_end=1178
_WORKERHEARTBEATRESPONSE._serialized_start=1181
_WORKERHEARTBEATRESPONSE._serialized_end=1314
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/full_type.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import enum_type_wrapper
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(tensorboard/compat/proto/full_type.proto\x12\x0btensorboard\"\x81\x01\n\x0b\x46ullTypeDef\x12(\n\x07type_id\x18\x01 \x01(\x0e\x32\x17.tensorboard.FullTypeId\x12&\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x18.tensorboard.FullTypeDef\x12\x0b\n\x01s\x18\x03 \x01(\tH\x00\x12\x0b\n\x01i\x18\x04 \x01(\x03H\x00\x42\x06\n\x04\x61ttr*\xda\x04\n\nFullTypeId\x12\r\n\tTFT_UNSET\x10\x00\x12\x0b\n\x07TFT_VAR\x10\x01\x12\x0b\n\x07TFT_ANY\x10\x02\x12\x0f\n\x0bTFT_PRODUCT\x10\x03\x12\r\n\tTFT_NAMED\x10\x04\x12\x10\n\x0cTFT_FOR_EACH\x10\x14\x12\x10\n\x0cTFT_CALLABLE\x10\x64\x12\x0f\n\nTFT_TENSOR\x10\xe8\x07\x12\x0e\n\tTFT_ARRAY\x10\xe9\x07\x12\x11\n\x0cTFT_OPTIONAL\x10\xea\x07\x12\x10\n\x0bTFT_LITERAL\x10\xeb\x07\x12\x10\n\x0bTFT_ENCODED\x10\xec\x07\x12\x15\n\x10TFT_SHAPE_TENSOR\x10\xed\x07\x12\r\n\x08TFT_BOOL\x10\xc8\x01\x12\x0e\n\tTFT_UINT8\x10\xc9\x01\x12\x0f\n\nTFT_UINT16\x10\xca\x01\x12\x0f\n\nTFT_UINT32\x10\xcb\x01\x12\x0f\n\nTFT_UINT64\x10\xcc\x01\x12\r\n\x08TFT_INT8\x10\xcd\x01\x12\x0e\n\tTFT_INT16\x10\xce\x01\x12\x0e\n\tTFT_INT32\x10\xcf\x01\x12\x0e\n\tTFT_INT64\x10\xd0\x01\x12\r\n\x08TFT_HALF\x10\xd1\x01\x12\x0e\n\tTFT_FLOAT\x10\xd2\x01\x12\x0f\n\nTFT_DOUBLE\x10\xd3\x01\x12\x11\n\x0cTFT_BFLOAT16\x10\xd7\x01\x12\x12\n\rTFT_COMPLEX64\x10\xd4\x01\x12\x13\n\x0eTFT_COMPLEX128\x10\xd5\x01\x12\x0f\n\nTFT_STRING\x10\xd6\x01\x12\x10\n\x0bTFT_DATASET\x10\xf6N\x12\x0f\n\nTFT_RAGGED\x10\xf7N\x12\x11\n\x0cTFT_ITERATOR\x10\xf8N\x12\x13\n\x0eTFT_MUTEX_LOCK\x10\xdaO\x12\x17\n\x12TFT_LEGACY_VARIANT\x10\xdbOB\x81\x01\n\x18org.tensorflow.frameworkB\x0e\x46ullTypeProtosP\x01ZPgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/full_type_go_proto\xf8\x01\x01\x62\x06proto3')
_FULLTYPEID = DESCRIPTOR.enum_types_by_name['FullTypeId']
FullTypeId = enum_type_wrapper.EnumTypeWrapper(_FULLTYPEID)
TFT_UNSET = 0
TFT_VAR = 1
TFT_ANY = 2
TFT_PRODUCT = 3
TFT_NAMED = 4
TFT_FOR_EACH = 20
TFT_CALLABLE = 100
TFT_TENSOR = 1000
TFT_ARRAY = 1001
TFT_OPTIONAL = 1002
TFT_LITERAL = 1003
TFT_ENCODED = 1004
TFT_SHAPE_TENSOR = 1005
TFT_BOOL = 200
TFT_UINT8 = 201
TFT_UINT16 = 202
TFT_UINT32 = 203
TFT_UINT64 = 204
TFT_INT8 = 205
TFT_INT16 = 206
TFT_INT32 = 207
TFT_INT64 = 208
TFT_HALF = 209
TFT_FLOAT = 210
TFT_DOUBLE = 211
TFT_BFLOAT16 = 215
TFT_COMPLEX64 = 212
TFT_COMPLEX128 = 213
TFT_STRING = 214
TFT_DATASET = 10102
TFT_RAGGED = 10103
TFT_ITERATOR = 10104
TFT_MUTEX_LOCK = 10202
TFT_LEGACY_VARIANT = 10203
_FULLTYPEDEF = DESCRIPTOR.message_types_by_name['FullTypeDef']
FullTypeDef = _reflection.GeneratedProtocolMessageType('FullTypeDef', (_message.Message,), {
'DESCRIPTOR' : _FULLTYPEDEF,
'__module__' : 'tensorboard.compat.proto.full_type_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FullTypeDef)
})
_sym_db.RegisterMessage(FullTypeDef)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\016FullTypeProtosP\001ZPgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/full_type_go_proto\370\001\001'
_FULLTYPEID._serialized_start=190
_FULLTYPEID._serialized_end=792
_FULLTYPEDEF._serialized_start=58
_FULLTYPEDEF._serialized_end=187
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,157 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/function.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import attr_value_pb2 as tensorboard_dot_compat_dot_proto_dot_attr__value__pb2
from tensorboard.compat.proto import node_def_pb2 as tensorboard_dot_compat_dot_proto_dot_node__def__pb2
from tensorboard.compat.proto import op_def_pb2 as tensorboard_dot_compat_dot_proto_dot_op__def__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\'tensorboard/compat/proto/function.proto\x12\x0btensorboard\x1a)tensorboard/compat/proto/attr_value.proto\x1a\'tensorboard/compat/proto/node_def.proto\x1a%tensorboard/compat/proto/op_def.proto\"\xab\x01\n\x12\x46unctionDefLibrary\x12*\n\x08\x66unction\x18\x01 \x03(\x0b\x32\x18.tensorboard.FunctionDef\x12*\n\x08gradient\x18\x02 \x03(\x0b\x32\x18.tensorboard.GradientDef\x12=\n\x14registered_gradients\x18\x03 \x03(\x0b\x32\x1f.tensorboard.RegisteredGradient\"\xcf\x06\n\x0b\x46unctionDef\x12%\n\tsignature\x18\x01 \x01(\x0b\x32\x12.tensorboard.OpDef\x12\x30\n\x04\x61ttr\x18\x05 \x03(\x0b\x32\".tensorboard.FunctionDef.AttrEntry\x12\x37\n\x08\x61rg_attr\x18\x07 \x03(\x0b\x32%.tensorboard.FunctionDef.ArgAttrEntry\x12Q\n\x16resource_arg_unique_id\x18\x08 \x03(\x0b\x32\x31.tensorboard.FunctionDef.ResourceArgUniqueIdEntry\x12&\n\x08node_def\x18\x03 \x03(\x0b\x32\x14.tensorboard.NodeDef\x12.\n\x03ret\x18\x04 \x03(\x0b\x32!.tensorboard.FunctionDef.RetEntry\x12=\n\x0b\x63ontrol_ret\x18\x06 \x03(\x0b\x32(.tensorboard.FunctionDef.ControlRetEntry\x1a\x43\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.tensorboard.AttrValue:\x02\x38\x01\x1a\x8a\x01\n\x08\x41rgAttrs\x12\x39\n\x04\x61ttr\x18\x01 \x03(\x0b\x32+.tensorboard.FunctionDef.ArgAttrs.AttrEntry\x1a\x43\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.tensorboard.AttrValue:\x02\x38\x01\x1aQ\n\x0c\x41rgAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12\x30\n\x05value\x18\x02 \x01(\x0b\x32!.tensorboard.FunctionDef.ArgAttrs:\x02\x38\x01\x1a:\n\x18ResourceArgUniqueIdEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12\r\n\x05value\x18\x02 \x01(\r:\x02\x38\x01\x1a*\n\x08RetEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0f\x43ontrolRetEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x02\x10\x03\";\n\x0bGradientDef\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x15\n\rgradient_func\x18\x02 \x01(\t\"G\n\x12RegisteredGradient\x12\x15\n\rgradient_func\x18\x01 \x01(\t\x12\x1a\n\x12registered_op_type\x18\x02 \x01(\tB\x80\x01\n\x18org.tensorflow.frameworkB\x0e\x46unctionProtosP\x01ZOgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/function_go_proto\xf8\x01\x01\x62\x06proto3')
_FUNCTIONDEFLIBRARY = DESCRIPTOR.message_types_by_name['FunctionDefLibrary']
_FUNCTIONDEF = DESCRIPTOR.message_types_by_name['FunctionDef']
_FUNCTIONDEF_ATTRENTRY = _FUNCTIONDEF.nested_types_by_name['AttrEntry']
_FUNCTIONDEF_ARGATTRS = _FUNCTIONDEF.nested_types_by_name['ArgAttrs']
_FUNCTIONDEF_ARGATTRS_ATTRENTRY = _FUNCTIONDEF_ARGATTRS.nested_types_by_name['AttrEntry']
_FUNCTIONDEF_ARGATTRENTRY = _FUNCTIONDEF.nested_types_by_name['ArgAttrEntry']
_FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY = _FUNCTIONDEF.nested_types_by_name['ResourceArgUniqueIdEntry']
_FUNCTIONDEF_RETENTRY = _FUNCTIONDEF.nested_types_by_name['RetEntry']
_FUNCTIONDEF_CONTROLRETENTRY = _FUNCTIONDEF.nested_types_by_name['ControlRetEntry']
_GRADIENTDEF = DESCRIPTOR.message_types_by_name['GradientDef']
_REGISTEREDGRADIENT = DESCRIPTOR.message_types_by_name['RegisteredGradient']
FunctionDefLibrary = _reflection.GeneratedProtocolMessageType('FunctionDefLibrary', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEFLIBRARY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDefLibrary)
})
_sym_db.RegisterMessage(FunctionDefLibrary)
FunctionDef = _reflection.GeneratedProtocolMessageType('FunctionDef', (_message.Message,), {
'AttrEntry' : _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_ATTRENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.AttrEntry)
})
,
'ArgAttrs' : _reflection.GeneratedProtocolMessageType('ArgAttrs', (_message.Message,), {
'AttrEntry' : _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_ARGATTRS_ATTRENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.ArgAttrs.AttrEntry)
})
,
'DESCRIPTOR' : _FUNCTIONDEF_ARGATTRS,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.ArgAttrs)
})
,
'ArgAttrEntry' : _reflection.GeneratedProtocolMessageType('ArgAttrEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_ARGATTRENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.ArgAttrEntry)
})
,
'ResourceArgUniqueIdEntry' : _reflection.GeneratedProtocolMessageType('ResourceArgUniqueIdEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.ResourceArgUniqueIdEntry)
})
,
'RetEntry' : _reflection.GeneratedProtocolMessageType('RetEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_RETENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.RetEntry)
})
,
'ControlRetEntry' : _reflection.GeneratedProtocolMessageType('ControlRetEntry', (_message.Message,), {
'DESCRIPTOR' : _FUNCTIONDEF_CONTROLRETENTRY,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef.ControlRetEntry)
})
,
'DESCRIPTOR' : _FUNCTIONDEF,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.FunctionDef)
})
_sym_db.RegisterMessage(FunctionDef)
_sym_db.RegisterMessage(FunctionDef.AttrEntry)
_sym_db.RegisterMessage(FunctionDef.ArgAttrs)
_sym_db.RegisterMessage(FunctionDef.ArgAttrs.AttrEntry)
_sym_db.RegisterMessage(FunctionDef.ArgAttrEntry)
_sym_db.RegisterMessage(FunctionDef.ResourceArgUniqueIdEntry)
_sym_db.RegisterMessage(FunctionDef.RetEntry)
_sym_db.RegisterMessage(FunctionDef.ControlRetEntry)
GradientDef = _reflection.GeneratedProtocolMessageType('GradientDef', (_message.Message,), {
'DESCRIPTOR' : _GRADIENTDEF,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GradientDef)
})
_sym_db.RegisterMessage(GradientDef)
RegisteredGradient = _reflection.GeneratedProtocolMessageType('RegisteredGradient', (_message.Message,), {
'DESCRIPTOR' : _REGISTEREDGRADIENT,
'__module__' : 'tensorboard.compat.proto.function_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RegisteredGradient)
})
_sym_db.RegisterMessage(RegisteredGradient)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\016FunctionProtosP\001ZOgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/function_go_proto\370\001\001'
_FUNCTIONDEF_ATTRENTRY._options = None
_FUNCTIONDEF_ATTRENTRY._serialized_options = b'8\001'
_FUNCTIONDEF_ARGATTRS_ATTRENTRY._options = None
_FUNCTIONDEF_ARGATTRS_ATTRENTRY._serialized_options = b'8\001'
_FUNCTIONDEF_ARGATTRENTRY._options = None
_FUNCTIONDEF_ARGATTRENTRY._serialized_options = b'8\001'
_FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY._options = None
_FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY._serialized_options = b'8\001'
_FUNCTIONDEF_RETENTRY._options = None
_FUNCTIONDEF_RETENTRY._serialized_options = b'8\001'
_FUNCTIONDEF_CONTROLRETENTRY._options = None
_FUNCTIONDEF_CONTROLRETENTRY._serialized_options = b'8\001'
_FUNCTIONDEFLIBRARY._serialized_start=180
_FUNCTIONDEFLIBRARY._serialized_end=351
_FUNCTIONDEF._serialized_start=354
_FUNCTIONDEF._serialized_end=1201
_FUNCTIONDEF_ATTRENTRY._serialized_start=749
_FUNCTIONDEF_ATTRENTRY._serialized_end=816
_FUNCTIONDEF_ARGATTRS._serialized_start=819
_FUNCTIONDEF_ARGATTRS._serialized_end=957
_FUNCTIONDEF_ARGATTRS_ATTRENTRY._serialized_start=749
_FUNCTIONDEF_ARGATTRS_ATTRENTRY._serialized_end=816
_FUNCTIONDEF_ARGATTRENTRY._serialized_start=959
_FUNCTIONDEF_ARGATTRENTRY._serialized_end=1040
_FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY._serialized_start=1042
_FUNCTIONDEF_RESOURCEARGUNIQUEIDENTRY._serialized_end=1100
_FUNCTIONDEF_RETENTRY._serialized_start=1102
_FUNCTIONDEF_RETENTRY._serialized_end=1144
_FUNCTIONDEF_CONTROLRETENTRY._serialized_start=1146
_FUNCTIONDEF_CONTROLRETENTRY._serialized_end=1195
_GRADIENTDEF._serialized_start=1203
_GRADIENTDEF._serialized_end=1262
_REGISTEREDGRADIENT._serialized_start=1264
_REGISTEREDGRADIENT._serialized_end=1335
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/graph_debug_info.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n/tensorboard/compat/proto/graph_debug_info.proto\x12\x0btensorboard\"\xab\x06\n\x0eGraphDebugInfo\x12\r\n\x05\x66iles\x18\x01 \x03(\t\x12\x41\n\x0c\x66rames_by_id\x18\x04 \x03(\x0b\x32+.tensorboard.GraphDebugInfo.FramesByIdEntry\x12\x41\n\x0ctraces_by_id\x18\x06 \x03(\x0b\x32+.tensorboard.GraphDebugInfo.TracesByIdEntry\x12\x37\n\x06traces\x18\x02 \x03(\x0b\x32\'.tensorboard.GraphDebugInfo.TracesEntry\x12H\n\x10name_to_trace_id\x18\x05 \x03(\x0b\x32..tensorboard.GraphDebugInfo.NameToTraceIdEntry\x1aX\n\x0b\x46ileLineCol\x12\x12\n\nfile_index\x18\x01 \x01(\x05\x12\x0c\n\x04line\x18\x02 \x01(\x05\x12\x0b\n\x03\x63ol\x18\x03 \x01(\x05\x12\x0c\n\x04\x66unc\x18\x04 \x01(\t\x12\x0c\n\x04\x63ode\x18\x05 \x01(\t\x1a\x63\n\nStackTrace\x12?\n\x0e\x66ile_line_cols\x18\x01 \x03(\x0b\x32\'.tensorboard.GraphDebugInfo.FileLineCol\x12\x14\n\x08\x66rame_id\x18\x02 \x03(\x06\x42\x02\x10\x01\x1aZ\n\x0f\x46ramesByIdEntry\x12\x0b\n\x03key\x18\x01 \x01(\x06\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.tensorboard.GraphDebugInfo.FileLineCol:\x02\x38\x01\x1aY\n\x0fTracesByIdEntry\x12\x0b\n\x03key\x18\x01 \x01(\x06\x12\x35\n\x05value\x18\x02 \x01(\x0b\x32&.tensorboard.GraphDebugInfo.StackTrace:\x02\x38\x01\x1aU\n\x0bTracesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x35\n\x05value\x18\x02 \x01(\x0b\x32&.tensorboard.GraphDebugInfo.StackTrace:\x02\x38\x01\x1a\x34\n\x12NameToTraceIdEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x06:\x02\x38\x01\x42\x8c\x01\n\x18org.tensorflow.frameworkB\x14GraphDebugInfoProtosP\x01ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\xf8\x01\x01')
_GRAPHDEBUGINFO = DESCRIPTOR.message_types_by_name['GraphDebugInfo']
_GRAPHDEBUGINFO_FILELINECOL = _GRAPHDEBUGINFO.nested_types_by_name['FileLineCol']
_GRAPHDEBUGINFO_STACKTRACE = _GRAPHDEBUGINFO.nested_types_by_name['StackTrace']
_GRAPHDEBUGINFO_FRAMESBYIDENTRY = _GRAPHDEBUGINFO.nested_types_by_name['FramesByIdEntry']
_GRAPHDEBUGINFO_TRACESBYIDENTRY = _GRAPHDEBUGINFO.nested_types_by_name['TracesByIdEntry']
_GRAPHDEBUGINFO_TRACESENTRY = _GRAPHDEBUGINFO.nested_types_by_name['TracesEntry']
_GRAPHDEBUGINFO_NAMETOTRACEIDENTRY = _GRAPHDEBUGINFO.nested_types_by_name['NameToTraceIdEntry']
GraphDebugInfo = _reflection.GeneratedProtocolMessageType('GraphDebugInfo', (_message.Message,), {
'FileLineCol' : _reflection.GeneratedProtocolMessageType('FileLineCol', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_FILELINECOL,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.FileLineCol)
})
,
'StackTrace' : _reflection.GeneratedProtocolMessageType('StackTrace', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_STACKTRACE,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.StackTrace)
})
,
'FramesByIdEntry' : _reflection.GeneratedProtocolMessageType('FramesByIdEntry', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_FRAMESBYIDENTRY,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.FramesByIdEntry)
})
,
'TracesByIdEntry' : _reflection.GeneratedProtocolMessageType('TracesByIdEntry', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_TRACESBYIDENTRY,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.TracesByIdEntry)
})
,
'TracesEntry' : _reflection.GeneratedProtocolMessageType('TracesEntry', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_TRACESENTRY,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.TracesEntry)
})
,
'NameToTraceIdEntry' : _reflection.GeneratedProtocolMessageType('NameToTraceIdEntry', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEBUGINFO_NAMETOTRACEIDENTRY,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo.NameToTraceIdEntry)
})
,
'DESCRIPTOR' : _GRAPHDEBUGINFO,
'__module__' : 'tensorboard.compat.proto.graph_debug_info_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDebugInfo)
})
_sym_db.RegisterMessage(GraphDebugInfo)
_sym_db.RegisterMessage(GraphDebugInfo.FileLineCol)
_sym_db.RegisterMessage(GraphDebugInfo.StackTrace)
_sym_db.RegisterMessage(GraphDebugInfo.FramesByIdEntry)
_sym_db.RegisterMessage(GraphDebugInfo.TracesByIdEntry)
_sym_db.RegisterMessage(GraphDebugInfo.TracesEntry)
_sym_db.RegisterMessage(GraphDebugInfo.NameToTraceIdEntry)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\024GraphDebugInfoProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_GRAPHDEBUGINFO_STACKTRACE.fields_by_name['frame_id']._options = None
_GRAPHDEBUGINFO_STACKTRACE.fields_by_name['frame_id']._serialized_options = b'\020\001'
_GRAPHDEBUGINFO_FRAMESBYIDENTRY._options = None
_GRAPHDEBUGINFO_FRAMESBYIDENTRY._serialized_options = b'8\001'
_GRAPHDEBUGINFO_TRACESBYIDENTRY._options = None
_GRAPHDEBUGINFO_TRACESBYIDENTRY._serialized_options = b'8\001'
_GRAPHDEBUGINFO_TRACESENTRY._options = None
_GRAPHDEBUGINFO_TRACESENTRY._serialized_options = b'8\001'
_GRAPHDEBUGINFO_NAMETOTRACEIDENTRY._options = None
_GRAPHDEBUGINFO_NAMETOTRACEIDENTRY._serialized_options = b'8\001'
_GRAPHDEBUGINFO._serialized_start=65
_GRAPHDEBUGINFO._serialized_end=876
_GRAPHDEBUGINFO_FILELINECOL._serialized_start=363
_GRAPHDEBUGINFO_FILELINECOL._serialized_end=451
_GRAPHDEBUGINFO_STACKTRACE._serialized_start=453
_GRAPHDEBUGINFO_STACKTRACE._serialized_end=552
_GRAPHDEBUGINFO_FRAMESBYIDENTRY._serialized_start=554
_GRAPHDEBUGINFO_FRAMESBYIDENTRY._serialized_end=644
_GRAPHDEBUGINFO_TRACESBYIDENTRY._serialized_start=646
_GRAPHDEBUGINFO_TRACESBYIDENTRY._serialized_end=735
_GRAPHDEBUGINFO_TRACESENTRY._serialized_start=737
_GRAPHDEBUGINFO_TRACESENTRY._serialized_end=822
_GRAPHDEBUGINFO_NAMETOTRACEIDENTRY._serialized_start=824
_GRAPHDEBUGINFO_NAMETOTRACEIDENTRY._serialized_end=876
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/graph.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import function_pb2 as tensorboard_dot_compat_dot_proto_dot_function__pb2
from tensorboard.compat.proto import graph_debug_info_pb2 as tensorboard_dot_compat_dot_proto_dot_graph__debug__info__pb2
from tensorboard.compat.proto import node_def_pb2 as tensorboard_dot_compat_dot_proto_dot_node__def__pb2
from tensorboard.compat.proto import versions_pb2 as tensorboard_dot_compat_dot_proto_dot_versions__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$tensorboard/compat/proto/graph.proto\x12\x0btensorboard\x1a\'tensorboard/compat/proto/function.proto\x1a/tensorboard/compat/proto/graph_debug_info.proto\x1a\'tensorboard/compat/proto/node_def.proto\x1a\'tensorboard/compat/proto/versions.proto\"\xd1\x01\n\x08GraphDef\x12\"\n\x04node\x18\x01 \x03(\x0b\x32\x14.tensorboard.NodeDef\x12)\n\x08versions\x18\x04 \x01(\x0b\x32\x17.tensorboard.VersionDef\x12\x13\n\x07version\x18\x03 \x01(\x05\x42\x02\x18\x01\x12\x30\n\x07library\x18\x02 \x01(\x0b\x32\x1f.tensorboard.FunctionDefLibrary\x12/\n\ndebug_info\x18\x05 \x01(\x0b\x32\x1b.tensorboard.GraphDebugInfoBz\n\x18org.tensorflow.frameworkB\x0bGraphProtosP\x01ZLgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/graph_go_proto\xf8\x01\x01\x62\x06proto3')
_GRAPHDEF = DESCRIPTOR.message_types_by_name['GraphDef']
GraphDef = _reflection.GeneratedProtocolMessageType('GraphDef', (_message.Message,), {
'DESCRIPTOR' : _GRAPHDEF,
'__module__' : 'tensorboard.compat.proto.graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.GraphDef)
})
_sym_db.RegisterMessage(GraphDef)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\013GraphProtosP\001ZLgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/graph_go_proto\370\001\001'
_GRAPHDEF.fields_by_name['version']._options = None
_GRAPHDEF.fields_by_name['version']._serialized_options = b'\030\001'
_GRAPHDEF._serialized_start=226
_GRAPHDEF._serialized_end=435
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/histogram.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(tensorboard/compat/proto/histogram.proto\x12\x0btensorboard\"\x87\x01\n\x0eHistogramProto\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03num\x18\x03 \x01(\x01\x12\x0b\n\x03sum\x18\x04 \x01(\x01\x12\x13\n\x0bsum_squares\x18\x05 \x01(\x01\x12\x18\n\x0c\x62ucket_limit\x18\x06 \x03(\x01\x42\x02\x10\x01\x12\x12\n\x06\x62ucket\x18\x07 \x03(\x01\x42\x02\x10\x01\x42\\\n\x18org.tensorflow.frameworkP\x01Z;github.com/google/tsl/tsl/go/core/protobuf/summary_go_proto\xf8\x01\x01\x62\x06proto3')
_HISTOGRAMPROTO = DESCRIPTOR.message_types_by_name['HistogramProto']
HistogramProto = _reflection.GeneratedProtocolMessageType('HistogramProto', (_message.Message,), {
'DESCRIPTOR' : _HISTOGRAMPROTO,
'__module__' : 'tensorboard.compat.proto.histogram_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.HistogramProto)
})
_sym_db.RegisterMessage(HistogramProto)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkP\001Z;github.com/google/tsl/tsl/go/core/protobuf/summary_go_proto\370\001\001'
_HISTOGRAMPROTO.fields_by_name['bucket_limit']._options = None
_HISTOGRAMPROTO.fields_by_name['bucket_limit']._serialized_options = b'\020\001'
_HISTOGRAMPROTO.fields_by_name['bucket']._options = None
_HISTOGRAMPROTO.fields_by_name['bucket']._serialized_options = b'\020\001'
_HISTOGRAMPROTO._serialized_start=58
_HISTOGRAMPROTO._serialized_end=193
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,254 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/meta_graph.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
from tensorboard.compat.proto import graph_pb2 as tensorboard_dot_compat_dot_proto_dot_graph__pb2
from tensorboard.compat.proto import op_def_pb2 as tensorboard_dot_compat_dot_proto_dot_op__def__pb2
from tensorboard.compat.proto import tensor_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__pb2
from tensorboard.compat.proto import tensor_shape_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
from tensorboard.compat.proto import saved_object_graph_pb2 as tensorboard_dot_compat_dot_proto_dot_saved__object__graph__pb2
from tensorboard.compat.proto import saver_pb2 as tensorboard_dot_compat_dot_proto_dot_saver__pb2
from tensorboard.compat.proto import struct_pb2 as tensorboard_dot_compat_dot_proto_dot_struct__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)tensorboard/compat/proto/meta_graph.proto\x12\x0btensorboard\x1a\x19google/protobuf/any.proto\x1a$tensorboard/compat/proto/graph.proto\x1a%tensorboard/compat/proto/op_def.proto\x1a%tensorboard/compat/proto/tensor.proto\x1a+tensorboard/compat/proto/tensor_shape.proto\x1a$tensorboard/compat/proto/types.proto\x1a\x31tensorboard/compat/proto/saved_object_graph.proto\x1a$tensorboard/compat/proto/saver.proto\x1a%tensorboard/compat/proto/struct.proto\"\xb3\x07\n\x0cMetaGraphDef\x12<\n\rmeta_info_def\x18\x01 \x01(\x0b\x32%.tensorboard.MetaGraphDef.MetaInfoDef\x12(\n\tgraph_def\x18\x02 \x01(\x0b\x32\x15.tensorboard.GraphDef\x12(\n\tsaver_def\x18\x03 \x01(\x0b\x32\x15.tensorboard.SaverDef\x12\x44\n\x0e\x63ollection_def\x18\x04 \x03(\x0b\x32,.tensorboard.MetaGraphDef.CollectionDefEntry\x12\x42\n\rsignature_def\x18\x05 \x03(\x0b\x32+.tensorboard.MetaGraphDef.SignatureDefEntry\x12\x31\n\x0e\x61sset_file_def\x18\x06 \x03(\x0b\x32\x19.tensorboard.AssetFileDef\x12\x37\n\x10object_graph_def\x18\x07 \x01(\x0b\x32\x1d.tensorboard.SavedObjectGraph\x1a\xf8\x02\n\x0bMetaInfoDef\x12\x1a\n\x12meta_graph_version\x18\x01 \x01(\t\x12-\n\x10stripped_op_list\x18\x02 \x01(\x0b\x32\x13.tensorboard.OpList\x12&\n\x08\x61ny_info\x18\x03 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x1a\n\x12tensorflow_version\x18\x05 \x01(\t\x12\x1e\n\x16tensorflow_git_version\x18\x06 \x01(\t\x12\x1e\n\x16stripped_default_attrs\x18\x07 \x01(\x08\x12T\n\x10\x66unction_aliases\x18\x08 \x03(\x0b\x32:.tensorboard.MetaGraphDef.MetaInfoDef.FunctionAliasesEntry\x1a\x36\n\x14\x46unctionAliasesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aP\n\x12\x43ollectionDefEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.tensorboard.CollectionDef:\x02\x38\x01\x1aN\n\x11SignatureDefEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.tensorboard.SignatureDef:\x02\x38\x01\"\xe4\x03\n\rCollectionDef\x12\x38\n\tnode_list\x18\x01 \x01(\x0b\x32#.tensorboard.CollectionDef.NodeListH\x00\x12:\n\nbytes_list\x18\x02 \x01(\x0b\x32$.tensorboard.CollectionDef.BytesListH\x00\x12:\n\nint64_list\x18\x03 \x01(\x0b\x32$.tensorboard.CollectionDef.Int64ListH\x00\x12:\n\nfloat_list\x18\x04 \x01(\x0b\x32$.tensorboard.CollectionDef.FloatListH\x00\x12\x36\n\x08\x61ny_list\x18\x05 \x01(\x0b\x32\".tensorboard.CollectionDef.AnyListH\x00\x1a\x19\n\x08NodeList\x12\r\n\x05value\x18\x01 \x03(\t\x1a\x1a\n\tBytesList\x12\r\n\x05value\x18\x01 \x03(\x0c\x1a\x1e\n\tInt64List\x12\x11\n\x05value\x18\x01 \x03(\x03\x42\x02\x10\x01\x1a\x1e\n\tFloatList\x12\x11\n\x05value\x18\x01 \x03(\x02\x42\x02\x10\x01\x1a.\n\x07\x41nyList\x12#\n\x05value\x18\x01 \x03(\x0b\x32\x14.google.protobuf.AnyB\x06\n\x04kind\"\xd7\x03\n\nTensorInfo\x12\x0e\n\x04name\x18\x01 \x01(\tH\x00\x12\x37\n\ncoo_sparse\x18\x04 \x01(\x0b\x32!.tensorboard.TensorInfo.CooSparseH\x00\x12\x43\n\x10\x63omposite_tensor\x18\x05 \x01(\x0b\x32\'.tensorboard.TensorInfo.CompositeTensorH\x00\x12$\n\x05\x64type\x18\x02 \x01(\x0e\x32\x15.tensorboard.DataType\x12\x33\n\x0ctensor_shape\x18\x03 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProto\x1a\x65\n\tCooSparse\x12\x1a\n\x12values_tensor_name\x18\x01 \x01(\t\x12\x1b\n\x13indices_tensor_name\x18\x02 \x01(\t\x12\x1f\n\x17\x64\x65nse_shape_tensor_name\x18\x03 \x01(\t\x1am\n\x0f\x43ompositeTensor\x12-\n\ttype_spec\x18\x01 \x01(\x0b\x32\x1a.tensorboard.TypeSpecProto\x12+\n\ncomponents\x18\x02 \x03(\x0b\x32\x17.tensorboard.TensorInfoB\n\n\x08\x65ncoding\"\xaa\x03\n\x0cSignatureDef\x12\x35\n\x06inputs\x18\x01 \x03(\x0b\x32%.tensorboard.SignatureDef.InputsEntry\x12\x37\n\x07outputs\x18\x02 \x03(\x0b\x32&.tensorboard.SignatureDef.OutputsEntry\x12\x13\n\x0bmethod_name\x18\x03 \x01(\t\x12\x39\n\x08\x64\x65\x66\x61ults\x18\x04 \x03(\x0b\x32\'.tensorboard.SignatureDef.DefaultsEntry\x1a\x46\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tensorboard.TensorInfo:\x02\x38\x01\x1aG\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tensorboard.TensorInfo:\x02\x38\x01\x1aI\n\rDefaultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.tensorboard.TensorProto:\x02\x38\x01\"N\n\x0c\x41ssetFileDef\x12,\n\x0btensor_info\x18\x01 \x01(\x0b\x32\x17.tensorboard.TensorInfo\x12\x10\n\x08\x66ilename\x18\x02 \x01(\tB\x87\x01\n\x18org.tensorflow.frameworkB\x0fMetaGraphProtosP\x01ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\xf8\x01\x01\x62\x06proto3')
_METAGRAPHDEF = DESCRIPTOR.message_types_by_name['MetaGraphDef']
_METAGRAPHDEF_METAINFODEF = _METAGRAPHDEF.nested_types_by_name['MetaInfoDef']
_METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY = _METAGRAPHDEF_METAINFODEF.nested_types_by_name['FunctionAliasesEntry']
_METAGRAPHDEF_COLLECTIONDEFENTRY = _METAGRAPHDEF.nested_types_by_name['CollectionDefEntry']
_METAGRAPHDEF_SIGNATUREDEFENTRY = _METAGRAPHDEF.nested_types_by_name['SignatureDefEntry']
_COLLECTIONDEF = DESCRIPTOR.message_types_by_name['CollectionDef']
_COLLECTIONDEF_NODELIST = _COLLECTIONDEF.nested_types_by_name['NodeList']
_COLLECTIONDEF_BYTESLIST = _COLLECTIONDEF.nested_types_by_name['BytesList']
_COLLECTIONDEF_INT64LIST = _COLLECTIONDEF.nested_types_by_name['Int64List']
_COLLECTIONDEF_FLOATLIST = _COLLECTIONDEF.nested_types_by_name['FloatList']
_COLLECTIONDEF_ANYLIST = _COLLECTIONDEF.nested_types_by_name['AnyList']
_TENSORINFO = DESCRIPTOR.message_types_by_name['TensorInfo']
_TENSORINFO_COOSPARSE = _TENSORINFO.nested_types_by_name['CooSparse']
_TENSORINFO_COMPOSITETENSOR = _TENSORINFO.nested_types_by_name['CompositeTensor']
_SIGNATUREDEF = DESCRIPTOR.message_types_by_name['SignatureDef']
_SIGNATUREDEF_INPUTSENTRY = _SIGNATUREDEF.nested_types_by_name['InputsEntry']
_SIGNATUREDEF_OUTPUTSENTRY = _SIGNATUREDEF.nested_types_by_name['OutputsEntry']
_SIGNATUREDEF_DEFAULTSENTRY = _SIGNATUREDEF.nested_types_by_name['DefaultsEntry']
_ASSETFILEDEF = DESCRIPTOR.message_types_by_name['AssetFileDef']
MetaGraphDef = _reflection.GeneratedProtocolMessageType('MetaGraphDef', (_message.Message,), {
'MetaInfoDef' : _reflection.GeneratedProtocolMessageType('MetaInfoDef', (_message.Message,), {
'FunctionAliasesEntry' : _reflection.GeneratedProtocolMessageType('FunctionAliasesEntry', (_message.Message,), {
'DESCRIPTOR' : _METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.MetaGraphDef.MetaInfoDef.FunctionAliasesEntry)
})
,
'DESCRIPTOR' : _METAGRAPHDEF_METAINFODEF,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.MetaGraphDef.MetaInfoDef)
})
,
'CollectionDefEntry' : _reflection.GeneratedProtocolMessageType('CollectionDefEntry', (_message.Message,), {
'DESCRIPTOR' : _METAGRAPHDEF_COLLECTIONDEFENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.MetaGraphDef.CollectionDefEntry)
})
,
'SignatureDefEntry' : _reflection.GeneratedProtocolMessageType('SignatureDefEntry', (_message.Message,), {
'DESCRIPTOR' : _METAGRAPHDEF_SIGNATUREDEFENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.MetaGraphDef.SignatureDefEntry)
})
,
'DESCRIPTOR' : _METAGRAPHDEF,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.MetaGraphDef)
})
_sym_db.RegisterMessage(MetaGraphDef)
_sym_db.RegisterMessage(MetaGraphDef.MetaInfoDef)
_sym_db.RegisterMessage(MetaGraphDef.MetaInfoDef.FunctionAliasesEntry)
_sym_db.RegisterMessage(MetaGraphDef.CollectionDefEntry)
_sym_db.RegisterMessage(MetaGraphDef.SignatureDefEntry)
CollectionDef = _reflection.GeneratedProtocolMessageType('CollectionDef', (_message.Message,), {
'NodeList' : _reflection.GeneratedProtocolMessageType('NodeList', (_message.Message,), {
'DESCRIPTOR' : _COLLECTIONDEF_NODELIST,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef.NodeList)
})
,
'BytesList' : _reflection.GeneratedProtocolMessageType('BytesList', (_message.Message,), {
'DESCRIPTOR' : _COLLECTIONDEF_BYTESLIST,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef.BytesList)
})
,
'Int64List' : _reflection.GeneratedProtocolMessageType('Int64List', (_message.Message,), {
'DESCRIPTOR' : _COLLECTIONDEF_INT64LIST,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef.Int64List)
})
,
'FloatList' : _reflection.GeneratedProtocolMessageType('FloatList', (_message.Message,), {
'DESCRIPTOR' : _COLLECTIONDEF_FLOATLIST,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef.FloatList)
})
,
'AnyList' : _reflection.GeneratedProtocolMessageType('AnyList', (_message.Message,), {
'DESCRIPTOR' : _COLLECTIONDEF_ANYLIST,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef.AnyList)
})
,
'DESCRIPTOR' : _COLLECTIONDEF,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.CollectionDef)
})
_sym_db.RegisterMessage(CollectionDef)
_sym_db.RegisterMessage(CollectionDef.NodeList)
_sym_db.RegisterMessage(CollectionDef.BytesList)
_sym_db.RegisterMessage(CollectionDef.Int64List)
_sym_db.RegisterMessage(CollectionDef.FloatList)
_sym_db.RegisterMessage(CollectionDef.AnyList)
TensorInfo = _reflection.GeneratedProtocolMessageType('TensorInfo', (_message.Message,), {
'CooSparse' : _reflection.GeneratedProtocolMessageType('CooSparse', (_message.Message,), {
'DESCRIPTOR' : _TENSORINFO_COOSPARSE,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.TensorInfo.CooSparse)
})
,
'CompositeTensor' : _reflection.GeneratedProtocolMessageType('CompositeTensor', (_message.Message,), {
'DESCRIPTOR' : _TENSORINFO_COMPOSITETENSOR,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.TensorInfo.CompositeTensor)
})
,
'DESCRIPTOR' : _TENSORINFO,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.TensorInfo)
})
_sym_db.RegisterMessage(TensorInfo)
_sym_db.RegisterMessage(TensorInfo.CooSparse)
_sym_db.RegisterMessage(TensorInfo.CompositeTensor)
SignatureDef = _reflection.GeneratedProtocolMessageType('SignatureDef', (_message.Message,), {
'InputsEntry' : _reflection.GeneratedProtocolMessageType('InputsEntry', (_message.Message,), {
'DESCRIPTOR' : _SIGNATUREDEF_INPUTSENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SignatureDef.InputsEntry)
})
,
'OutputsEntry' : _reflection.GeneratedProtocolMessageType('OutputsEntry', (_message.Message,), {
'DESCRIPTOR' : _SIGNATUREDEF_OUTPUTSENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SignatureDef.OutputsEntry)
})
,
'DefaultsEntry' : _reflection.GeneratedProtocolMessageType('DefaultsEntry', (_message.Message,), {
'DESCRIPTOR' : _SIGNATUREDEF_DEFAULTSENTRY,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SignatureDef.DefaultsEntry)
})
,
'DESCRIPTOR' : _SIGNATUREDEF,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.SignatureDef)
})
_sym_db.RegisterMessage(SignatureDef)
_sym_db.RegisterMessage(SignatureDef.InputsEntry)
_sym_db.RegisterMessage(SignatureDef.OutputsEntry)
_sym_db.RegisterMessage(SignatureDef.DefaultsEntry)
AssetFileDef = _reflection.GeneratedProtocolMessageType('AssetFileDef', (_message.Message,), {
'DESCRIPTOR' : _ASSETFILEDEF,
'__module__' : 'tensorboard.compat.proto.meta_graph_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.AssetFileDef)
})
_sym_db.RegisterMessage(AssetFileDef)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\017MetaGraphProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY._options = None
_METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY._serialized_options = b'8\001'
_METAGRAPHDEF_COLLECTIONDEFENTRY._options = None
_METAGRAPHDEF_COLLECTIONDEFENTRY._serialized_options = b'8\001'
_METAGRAPHDEF_SIGNATUREDEFENTRY._options = None
_METAGRAPHDEF_SIGNATUREDEFENTRY._serialized_options = b'8\001'
_COLLECTIONDEF_INT64LIST.fields_by_name['value']._options = None
_COLLECTIONDEF_INT64LIST.fields_by_name['value']._serialized_options = b'\020\001'
_COLLECTIONDEF_FLOATLIST.fields_by_name['value']._options = None
_COLLECTIONDEF_FLOATLIST.fields_by_name['value']._serialized_options = b'\020\001'
_SIGNATUREDEF_INPUTSENTRY._options = None
_SIGNATUREDEF_INPUTSENTRY._serialized_options = b'8\001'
_SIGNATUREDEF_OUTPUTSENTRY._options = None
_SIGNATUREDEF_OUTPUTSENTRY._serialized_options = b'8\001'
_SIGNATUREDEF_DEFAULTSENTRY._options = None
_SIGNATUREDEF_DEFAULTSENTRY._serialized_options = b'8\001'
_METAGRAPHDEF._serialized_start=413
_METAGRAPHDEF._serialized_end=1360
_METAGRAPHDEF_METAINFODEF._serialized_start=822
_METAGRAPHDEF_METAINFODEF._serialized_end=1198
_METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY._serialized_start=1144
_METAGRAPHDEF_METAINFODEF_FUNCTIONALIASESENTRY._serialized_end=1198
_METAGRAPHDEF_COLLECTIONDEFENTRY._serialized_start=1200
_METAGRAPHDEF_COLLECTIONDEFENTRY._serialized_end=1280
_METAGRAPHDEF_SIGNATUREDEFENTRY._serialized_start=1282
_METAGRAPHDEF_SIGNATUREDEFENTRY._serialized_end=1360
_COLLECTIONDEF._serialized_start=1363
_COLLECTIONDEF._serialized_end=1847
_COLLECTIONDEF_NODELIST._serialized_start=1674
_COLLECTIONDEF_NODELIST._serialized_end=1699
_COLLECTIONDEF_BYTESLIST._serialized_start=1701
_COLLECTIONDEF_BYTESLIST._serialized_end=1727
_COLLECTIONDEF_INT64LIST._serialized_start=1729
_COLLECTIONDEF_INT64LIST._serialized_end=1759
_COLLECTIONDEF_FLOATLIST._serialized_start=1761
_COLLECTIONDEF_FLOATLIST._serialized_end=1791
_COLLECTIONDEF_ANYLIST._serialized_start=1793
_COLLECTIONDEF_ANYLIST._serialized_end=1839
_TENSORINFO._serialized_start=1850
_TENSORINFO._serialized_end=2321
_TENSORINFO_COOSPARSE._serialized_start=2097
_TENSORINFO_COOSPARSE._serialized_end=2198
_TENSORINFO_COMPOSITETENSOR._serialized_start=2200
_TENSORINFO_COMPOSITETENSOR._serialized_end=2309
_SIGNATUREDEF._serialized_start=2324
_SIGNATUREDEF._serialized_end=2750
_SIGNATUREDEF_INPUTSENTRY._serialized_start=2532
_SIGNATUREDEF_INPUTSENTRY._serialized_end=2602
_SIGNATUREDEF_OUTPUTSENTRY._serialized_start=2604
_SIGNATUREDEF_OUTPUTSENTRY._serialized_end=2675
_SIGNATUREDEF_DEFAULTSENTRY._serialized_start=2677
_SIGNATUREDEF_DEFAULTSENTRY._serialized_end=2750
_ASSETFILEDEF._serialized_start=2752
_ASSETFILEDEF._serialized_end=2830
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/node_def.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import attr_value_pb2 as tensorboard_dot_compat_dot_proto_dot_attr__value__pb2
from tensorboard.compat.proto import full_type_pb2 as tensorboard_dot_compat_dot_proto_dot_full__type__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\'tensorboard/compat/proto/node_def.proto\x12\x0btensorboard\x1a)tensorboard/compat/proto/attr_value.proto\x1a(tensorboard/compat/proto/full_type.proto\"\x8a\x03\n\x07NodeDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\n\n\x02op\x18\x02 \x01(\t\x12\r\n\x05input\x18\x03 \x03(\t\x12\x0e\n\x06\x64\x65vice\x18\x04 \x01(\t\x12,\n\x04\x61ttr\x18\x05 \x03(\x0b\x32\x1e.tensorboard.NodeDef.AttrEntry\x12K\n\x17\x65xperimental_debug_info\x18\x06 \x01(\x0b\x32*.tensorboard.NodeDef.ExperimentalDebugInfo\x12\x33\n\x11\x65xperimental_type\x18\x07 \x01(\x0b\x32\x18.tensorboard.FullTypeDef\x1a\x43\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.tensorboard.AttrValue:\x02\x38\x01\x1aQ\n\x15\x45xperimentalDebugInfo\x12\x1b\n\x13original_node_names\x18\x01 \x03(\t\x12\x1b\n\x13original_func_names\x18\x02 \x03(\tB{\n\x18org.tensorflow.frameworkB\tNodeProtoP\x01ZOgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/node_def_go_proto\xf8\x01\x01\x62\x06proto3')
_NODEDEF = DESCRIPTOR.message_types_by_name['NodeDef']
_NODEDEF_ATTRENTRY = _NODEDEF.nested_types_by_name['AttrEntry']
_NODEDEF_EXPERIMENTALDEBUGINFO = _NODEDEF.nested_types_by_name['ExperimentalDebugInfo']
NodeDef = _reflection.GeneratedProtocolMessageType('NodeDef', (_message.Message,), {
'AttrEntry' : _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), {
'DESCRIPTOR' : _NODEDEF_ATTRENTRY,
'__module__' : 'tensorboard.compat.proto.node_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.NodeDef.AttrEntry)
})
,
'ExperimentalDebugInfo' : _reflection.GeneratedProtocolMessageType('ExperimentalDebugInfo', (_message.Message,), {
'DESCRIPTOR' : _NODEDEF_EXPERIMENTALDEBUGINFO,
'__module__' : 'tensorboard.compat.proto.node_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.NodeDef.ExperimentalDebugInfo)
})
,
'DESCRIPTOR' : _NODEDEF,
'__module__' : 'tensorboard.compat.proto.node_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.NodeDef)
})
_sym_db.RegisterMessage(NodeDef)
_sym_db.RegisterMessage(NodeDef.AttrEntry)
_sym_db.RegisterMessage(NodeDef.ExperimentalDebugInfo)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\tNodeProtoP\001ZOgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/node_def_go_proto\370\001\001'
_NODEDEF_ATTRENTRY._options = None
_NODEDEF_ATTRENTRY._serialized_options = b'8\001'
_NODEDEF._serialized_start=142
_NODEDEF._serialized_end=536
_NODEDEF_ATTRENTRY._serialized_start=386
_NODEDEF_ATTRENTRY._serialized_end=453
_NODEDEF_EXPERIMENTALDEBUGINFO._serialized_start=455
_NODEDEF_EXPERIMENTALDEBUGINFO._serialized_end=536
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,81 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/op_def.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import attr_value_pb2 as tensorboard_dot_compat_dot_proto_dot_attr__value__pb2
from tensorboard.compat.proto import full_type_pb2 as tensorboard_dot_compat_dot_proto_dot_full__type__pb2
from tensorboard.compat.proto import resource_handle_pb2 as tensorboard_dot_compat_dot_proto_dot_resource__handle__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%tensorboard/compat/proto/op_def.proto\x12\x0btensorboard\x1a)tensorboard/compat/proto/attr_value.proto\x1a(tensorboard/compat/proto/full_type.proto\x1a.tensorboard/compat/proto/resource_handle.proto\x1a$tensorboard/compat/proto/types.proto\"\xfc\x06\n\x05OpDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\tinput_arg\x18\x02 \x03(\x0b\x32\x19.tensorboard.OpDef.ArgDef\x12-\n\noutput_arg\x18\x03 \x03(\x0b\x32\x19.tensorboard.OpDef.ArgDef\x12\x16\n\x0e\x63ontrol_output\x18\x14 \x03(\t\x12(\n\x04\x61ttr\x18\x04 \x03(\x0b\x32\x1a.tensorboard.OpDef.AttrDef\x12/\n\x0b\x64\x65precation\x18\x08 \x01(\x0b\x32\x1a.tensorboard.OpDeprecation\x12\x0f\n\x07summary\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x16\n\x0eis_commutative\x18\x12 \x01(\x08\x12\x14\n\x0cis_aggregate\x18\x10 \x01(\x08\x12\x13\n\x0bis_stateful\x18\x11 \x01(\x08\x12\"\n\x1a\x61llows_uninitialized_input\x18\x13 \x01(\x08\x12$\n\x1cis_distributed_communication\x18\x15 \x01(\x08\x1a\x9f\x02\n\x06\x41rgDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12#\n\x04type\x18\x03 \x01(\x0e\x32\x15.tensorboard.DataType\x12\x11\n\ttype_attr\x18\x04 \x01(\t\x12\x13\n\x0bnumber_attr\x18\x05 \x01(\t\x12\x16\n\x0etype_list_attr\x18\x06 \x01(\t\x12\x43\n\x0bhandle_data\x18\x07 \x03(\x0b\x32..tensorboard.ResourceHandleProto.DtypeAndShape\x12\x0e\n\x06is_ref\x18\x10 \x01(\x08\x12\x38\n\x16\x65xperimental_full_type\x18\x11 \x01(\x0b\x32\x18.tensorboard.FullTypeDef\x1a\xbf\x01\n\x07\x41ttrDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12-\n\rdefault_value\x18\x03 \x01(\x0b\x32\x16.tensorboard.AttrValue\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x13\n\x0bhas_minimum\x18\x05 \x01(\x08\x12\x0f\n\x07minimum\x18\x06 \x01(\x03\x12.\n\x0e\x61llowed_values\x18\x07 \x01(\x0b\x32\x16.tensorboard.AttrValue\"5\n\rOpDeprecation\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x13\n\x0b\x65xplanation\x18\x02 \x01(\t\"(\n\x06OpList\x12\x1e\n\x02op\x18\x01 \x03(\x0b\x32\x12.tensorboard.OpDefB{\n\x18org.tensorflow.frameworkB\x0bOpDefProtosP\x01ZMgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/op_def_go_proto\xf8\x01\x01\x62\x06proto3')
_OPDEF = DESCRIPTOR.message_types_by_name['OpDef']
_OPDEF_ARGDEF = _OPDEF.nested_types_by_name['ArgDef']
_OPDEF_ATTRDEF = _OPDEF.nested_types_by_name['AttrDef']
_OPDEPRECATION = DESCRIPTOR.message_types_by_name['OpDeprecation']
_OPLIST = DESCRIPTOR.message_types_by_name['OpList']
OpDef = _reflection.GeneratedProtocolMessageType('OpDef', (_message.Message,), {
'ArgDef' : _reflection.GeneratedProtocolMessageType('ArgDef', (_message.Message,), {
'DESCRIPTOR' : _OPDEF_ARGDEF,
'__module__' : 'tensorboard.compat.proto.op_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OpDef.ArgDef)
})
,
'AttrDef' : _reflection.GeneratedProtocolMessageType('AttrDef', (_message.Message,), {
'DESCRIPTOR' : _OPDEF_ATTRDEF,
'__module__' : 'tensorboard.compat.proto.op_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OpDef.AttrDef)
})
,
'DESCRIPTOR' : _OPDEF,
'__module__' : 'tensorboard.compat.proto.op_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OpDef)
})
_sym_db.RegisterMessage(OpDef)
_sym_db.RegisterMessage(OpDef.ArgDef)
_sym_db.RegisterMessage(OpDef.AttrDef)
OpDeprecation = _reflection.GeneratedProtocolMessageType('OpDeprecation', (_message.Message,), {
'DESCRIPTOR' : _OPDEPRECATION,
'__module__' : 'tensorboard.compat.proto.op_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OpDeprecation)
})
_sym_db.RegisterMessage(OpDeprecation)
OpList = _reflection.GeneratedProtocolMessageType('OpList', (_message.Message,), {
'DESCRIPTOR' : _OPLIST,
'__module__' : 'tensorboard.compat.proto.op_def_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.OpList)
})
_sym_db.RegisterMessage(OpList)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\013OpDefProtosP\001ZMgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/op_def_go_proto\370\001\001'
_OPDEF._serialized_start=226
_OPDEF._serialized_end=1118
_OPDEF_ARGDEF._serialized_start=637
_OPDEF_ARGDEF._serialized_end=924
_OPDEF_ATTRDEF._serialized_start=927
_OPDEF_ATTRDEF._serialized_end=1118
_OPDEPRECATION._serialized_start=1120
_OPDEPRECATION._serialized_end=1173
_OPLIST._serialized_start=1175
_OPLIST._serialized_end=1215
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/resource_handle.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import tensor_shape_pb2 as tensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2
from tensorboard.compat.proto import types_pb2 as tensorboard_dot_compat_dot_proto_dot_types__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.tensorboard/compat/proto/resource_handle.proto\x12\x0btensorboard\x1a+tensorboard/compat/proto/tensor_shape.proto\x1a$tensorboard/compat/proto/types.proto\"\xa8\x02\n\x13ResourceHandleProto\x12\x0e\n\x06\x64\x65vice\x18\x01 \x01(\t\x12\x11\n\tcontainer\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\thash_code\x18\x04 \x01(\x04\x12\x17\n\x0fmaybe_type_name\x18\x05 \x01(\t\x12I\n\x11\x64types_and_shapes\x18\x06 \x03(\x0b\x32..tensorboard.ResourceHandleProto.DtypeAndShape\x1a\x63\n\rDtypeAndShape\x12$\n\x05\x64type\x18\x01 \x01(\x0e\x32\x15.tensorboard.DataType\x12,\n\x05shape\x18\x02 \x01(\x0b\x32\x1d.tensorboard.TensorShapeProtoJ\x04\x08\x07\x10\x08\x42\x87\x01\n\x18org.tensorflow.frameworkB\x0eResourceHandleP\x01ZVgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto\xf8\x01\x01\x62\x06proto3')
_RESOURCEHANDLEPROTO = DESCRIPTOR.message_types_by_name['ResourceHandleProto']
_RESOURCEHANDLEPROTO_DTYPEANDSHAPE = _RESOURCEHANDLEPROTO.nested_types_by_name['DtypeAndShape']
ResourceHandleProto = _reflection.GeneratedProtocolMessageType('ResourceHandleProto', (_message.Message,), {
'DtypeAndShape' : _reflection.GeneratedProtocolMessageType('DtypeAndShape', (_message.Message,), {
'DESCRIPTOR' : _RESOURCEHANDLEPROTO_DTYPEANDSHAPE,
'__module__' : 'tensorboard.compat.proto.resource_handle_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ResourceHandleProto.DtypeAndShape)
})
,
'DESCRIPTOR' : _RESOURCEHANDLEPROTO,
'__module__' : 'tensorboard.compat.proto.resource_handle_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ResourceHandleProto)
})
_sym_db.RegisterMessage(ResourceHandleProto)
_sym_db.RegisterMessage(ResourceHandleProto.DtypeAndShape)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\016ResourceHandleP\001ZVgithub.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto\370\001\001'
_RESOURCEHANDLEPROTO._serialized_start=147
_RESOURCEHANDLEPROTO._serialized_end=443
_RESOURCEHANDLEPROTO_DTYPEANDSHAPE._serialized_start=338
_RESOURCEHANDLEPROTO_DTYPEANDSHAPE._serialized_end=437
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,93 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: tensorboard/compat/proto/rewriter_config.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from tensorboard.compat.proto import attr_value_pb2 as tensorboard_dot_compat_dot_proto_dot_attr__value__pb2
from tensorboard.compat.proto import verifier_config_pb2 as tensorboard_dot_compat_dot_proto_dot_verifier__config__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.tensorboard/compat/proto/rewriter_config.proto\x12\x0btensorboard\x1a)tensorboard/compat/proto/attr_value.proto\x1a.tensorboard/compat/proto/verifier_config.proto\";\n\x13\x41utoParallelOptions\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\x12\x14\n\x0cnum_replicas\x18\x02 \x01(\x05\"+\n\x16ScopedAllocatorOptions\x12\x11\n\tenable_op\x18\x01 \x03(\t\"\xb2\x16\n\x0eRewriterConfig\x12\x44\n\x15\x63pu_layout_conversion\x18\x32 \x01(\x0e\x32%.tensorboard.RewriterConfig.CpuLayout\x12<\n\x10layout_optimizer\x18\x01 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12<\n\x10\x63onstant_folding\x18\x03 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12>\n\x12shape_optimization\x18\r \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x35\n\tremapping\x18\x0e \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12G\n\x1b\x63ommon_subgraph_elimination\x18\x18 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x43\n\x17\x61rithmetic_optimization\x18\x07 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x43\n\x17\x64\x65pendency_optimization\x18\x08 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12=\n\x11loop_optimization\x18\t \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x41\n\x15\x66unction_optimization\x18\n \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12:\n\x0e\x64\x65\x62ug_stripper\x18\x0b \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x1d\n\x15\x64isable_model_pruning\x18\x02 \x01(\x08\x12I\n\x1dscoped_allocator_optimization\x18\x0f \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x44\n\x18pin_to_host_optimization\x18\x12 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x43\n\x17implementation_selector\x18\x16 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12@\n\x14\x61uto_mixed_precision\x18\x17 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x44\n\x18\x61uto_mixed_precision_mkl\x18\x19 \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12P\n$auto_mixed_precision_onednn_bfloat16\x18\x1f \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x44\n\x18\x61uto_mixed_precision_cpu\x18\x1d \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12\x1e\n\x16\x64isable_meta_optimizer\x18\x13 \x01(\x08\x12\x1d\n\x15\x64isable_tfg_optimizer\x18 \x01(\x08\x12\x41\n\x15use_plugin_optimizers\x18\x1c \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12P\n$experimental_conditional_code_motion\x18\x1e \x01(\x0e\x32\".tensorboard.RewriterConfig.Toggle\x12P\n\x19meta_optimizer_iterations\x18\x0c \x01(\x0e\x32-.tensorboard.RewriterConfig.NumIterationsType\x12\x17\n\x0fmin_graph_nodes\x18\x11 \x01(\x05\x12;\n3experimental_disable_compressed_tensor_optimization\x18\x1a \x01(\x08\x12;\n3experimental_disable_folding_quantization_emulation\x18\x1b \x01(\x08\x12\x43\n\x13memory_optimization\x18\x04 \x01(\x0e\x32&.tensorboard.RewriterConfig.MemOptType\x12/\n\'memory_optimizer_target_node_name_scope\x18\x06 \x01(\t\x12!\n\x19meta_optimizer_timeout_ms\x18\x14 \x01(\x03\x12\x37\n\rauto_parallel\x18\x05 \x01(\x0b\x32 .tensorboard.AutoParallelOptions\x12 \n\x18\x66\x61il_on_optimizer_errors\x18\x15 \x01(\x08\x12\x42\n\x15scoped_allocator_opts\x18\x10 \x01(\x0b\x32#.tensorboard.ScopedAllocatorOptions\x12\x12\n\noptimizers\x18\x64 \x03(\t\x12L\n\x11\x63ustom_optimizers\x18\xc8\x01 \x03(\x0b\x32\x30.tensorboard.RewriterConfig.CustomGraphOptimizer\x12\x45\n\x1finter_optimizer_verifier_config\x18\xac\x02 \x01(\x0b\x32\x1b.tensorboard.VerifierConfig\x12G\n!post_optimization_verifier_config\x18\xad\x02 \x01(\x0b\x32\x1b.tensorboard.VerifierConfig\x1a\xcc\x01\n\x14\x43ustomGraphOptimizer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12Y\n\rparameter_map\x18\x02 \x03(\x0b\x32\x42.tensorboard.RewriterConfig.CustomGraphOptimizer.ParameterMapEntry\x1aK\n\x11ParameterMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.tensorboard.AttrValue:\x02\x38\x01\"d\n\x06Toggle\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x06\n\x02ON\x10\x01\x12\x07\n\x03OFF\x10\x02\x12\x0e\n\nAGGRESSIVE\x10\x03\x12\x15\n\x11\x45XPERIMENTAL_MLIR\x10\x04\x12\x15\n\x11\x45XPERIMENTAL_BOTH\x10\x05\"I\n\tCpuLayout\x12\x18\n\x14NO_CONVERSION_ON_CPU\x10\x00\x12\x10\n\x0cNCHW_TO_NHWC\x10\x01\x12\x10\n\x0cNHWC_TO_NCHW\x10\x02\"<\n\x11NumIterationsType\x12\x15\n\x11\x44\x45\x46\x41ULT_NUM_ITERS\x10\x00\x12\x07\n\x03ONE\x10\x01\x12\x07\n\x03TWO\x10\x02\"\x9f\x01\n\nMemOptType\x12\x13\n\x0f\x44\x45\x46\x41ULT_MEM_OPT\x10\x00\x12\x0e\n\nNO_MEM_OPT\x10\x01\x12\n\n\x06MANUAL\x10\x02\x12\x17\n\x13SWAPPING_HEURISTICS\x10\x04\x12\x1c\n\x18RECOMPUTATION_HEURISTICS\x10\x05\x12\x19\n\x15SCHEDULING_HEURISTICS\x10\x06\x12\x0e\n\nHEURISTICS\x10\x03\x42\x8c\x01\n\x18org.tensorflow.frameworkB\x14RewriterConfigProtosP\x01ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\xf8\x01\x01\x62\x06proto3')
_AUTOPARALLELOPTIONS = DESCRIPTOR.message_types_by_name['AutoParallelOptions']
_SCOPEDALLOCATOROPTIONS = DESCRIPTOR.message_types_by_name['ScopedAllocatorOptions']
_REWRITERCONFIG = DESCRIPTOR.message_types_by_name['RewriterConfig']
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER = _REWRITERCONFIG.nested_types_by_name['CustomGraphOptimizer']
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY = _REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER.nested_types_by_name['ParameterMapEntry']
_REWRITERCONFIG_TOGGLE = _REWRITERCONFIG.enum_types_by_name['Toggle']
_REWRITERCONFIG_CPULAYOUT = _REWRITERCONFIG.enum_types_by_name['CpuLayout']
_REWRITERCONFIG_NUMITERATIONSTYPE = _REWRITERCONFIG.enum_types_by_name['NumIterationsType']
_REWRITERCONFIG_MEMOPTTYPE = _REWRITERCONFIG.enum_types_by_name['MemOptType']
AutoParallelOptions = _reflection.GeneratedProtocolMessageType('AutoParallelOptions', (_message.Message,), {
'DESCRIPTOR' : _AUTOPARALLELOPTIONS,
'__module__' : 'tensorboard.compat.proto.rewriter_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.AutoParallelOptions)
})
_sym_db.RegisterMessage(AutoParallelOptions)
ScopedAllocatorOptions = _reflection.GeneratedProtocolMessageType('ScopedAllocatorOptions', (_message.Message,), {
'DESCRIPTOR' : _SCOPEDALLOCATOROPTIONS,
'__module__' : 'tensorboard.compat.proto.rewriter_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.ScopedAllocatorOptions)
})
_sym_db.RegisterMessage(ScopedAllocatorOptions)
RewriterConfig = _reflection.GeneratedProtocolMessageType('RewriterConfig', (_message.Message,), {
'CustomGraphOptimizer' : _reflection.GeneratedProtocolMessageType('CustomGraphOptimizer', (_message.Message,), {
'ParameterMapEntry' : _reflection.GeneratedProtocolMessageType('ParameterMapEntry', (_message.Message,), {
'DESCRIPTOR' : _REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY,
'__module__' : 'tensorboard.compat.proto.rewriter_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RewriterConfig.CustomGraphOptimizer.ParameterMapEntry)
})
,
'DESCRIPTOR' : _REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER,
'__module__' : 'tensorboard.compat.proto.rewriter_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RewriterConfig.CustomGraphOptimizer)
})
,
'DESCRIPTOR' : _REWRITERCONFIG,
'__module__' : 'tensorboard.compat.proto.rewriter_config_pb2'
# @@protoc_insertion_point(class_scope:tensorboard.RewriterConfig)
})
_sym_db.RegisterMessage(RewriterConfig)
_sym_db.RegisterMessage(RewriterConfig.CustomGraphOptimizer)
_sym_db.RegisterMessage(RewriterConfig.CustomGraphOptimizer.ParameterMapEntry)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\030org.tensorflow.frameworkB\024RewriterConfigProtosP\001ZUgithub.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto\370\001\001'
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY._options = None
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY._serialized_options = b'8\001'
_AUTOPARALLELOPTIONS._serialized_start=154
_AUTOPARALLELOPTIONS._serialized_end=213
_SCOPEDALLOCATOROPTIONS._serialized_start=215
_SCOPEDALLOCATOROPTIONS._serialized_end=258
_REWRITERCONFIG._serialized_start=261
_REWRITERCONFIG._serialized_end=3127
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER._serialized_start=2522
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER._serialized_end=2726
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY._serialized_start=2651
_REWRITERCONFIG_CUSTOMGRAPHOPTIMIZER_PARAMETERMAPENTRY._serialized_end=2726
_REWRITERCONFIG_TOGGLE._serialized_start=2728
_REWRITERCONFIG_TOGGLE._serialized_end=2828
_REWRITERCONFIG_CPULAYOUT._serialized_start=2830
_REWRITERCONFIG_CPULAYOUT._serialized_end=2903
_REWRITERCONFIG_NUMITERATIONSTYPE._serialized_start=2905
_REWRITERCONFIG_NUMITERATIONSTYPE._serialized_end=2965
_REWRITERCONFIG_MEMOPTTYPE._serialized_start=2968
_REWRITERCONFIG_MEMOPTTYPE._serialized_end=3127
# @@protoc_insertion_point(module_scope)
Loaded 100 of 20524 files, more files were not shown because too many files have changed in this diff. Show more