引用の書式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75const CITATION_START = "\uE200";
const CITATION_DELIMITER = "\uE202";
const CITATION_STOP = "\uE201";

const SOURCE_ID_RE = /^[A-Za-z0-9_-]+$/;
const LINE_LOCATOR_RE = /^L\d+(?:-L\d+)?$/;

// Extract citations such as:
//
//   {CITATION_START}cite{CITATION_DELIMITER}turn0file0{CITATION_STOP}
//   {CITATION_START}cite{CITATION_DELIMITER}turn0file0{CITATION_DELIMITER}L8-L13{CITATION_STOP}
//   {CITATION_START}cite{CITATION_DELIMITER}turn0search0{CITATION_DELIMITER}turn1news2{CITATION_STOP}
function extractCitations(text, { families = ["cite"] } = {}) {
  if (families.length === 0) {
    return [];
  }

  const familyPattern = families
    .map((family) => family.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
    .join("|");

  const tokenRe = new RegExp(
    `${CITATION_START}(?<family>${familyPattern})${CITATION_DELIMITER}(?<body>[\\s\\S]*?)${CITATION_STOP}`,
    "g"
  );

  const citations = [];

  for (const match of text.matchAll(tokenRe)) {
    const body = match.groups?.body ?? "";
    const parts = body
      .split(CITATION_DELIMITER)
      .map((part) => part.trim())
      .filter(Boolean);

    if (parts.length === 0) {
      continue;
    }

    let locator = null;
    const lastPart = parts[parts.length - 1];
    if (LINE_LOCATOR_RE.test(lastPart)) {
      locator = parts.pop() ?? null;
    }

    if (parts.length === 0 || parts.some((part) => !SOURCE_ID_RE.test(part))) {
      continue;
    }

    citations.push({
      raw: match[0],
      family: match.groups?.family ?? "",
      source_ids: parts,
      locator,
      start: match.index ?? 0,
      end: (match.index ?? 0) + match[0].length,
    });
  }

  return citations;
}

function stripCitations(text, citations) {
  let cleanText = text;
  const sortedCitations = Array.from(citations).sort(
    (left, right) => right.start - left.start
  );

  for (const citation of sortedCitations) {
    cleanText =
      cleanText.slice(0, citation.start) + cleanText.slice(citation.end);
  }

  return cleanText;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63// Start and End are UTF-8 byte offsets, suitable for Go string slicing.
type Citation struct {
	Raw, Family string
	SourceIDs   []string
	Locator     string
	Start, End  int
}

var sourceID = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
var lineLocator = regexp.MustCompile(`^L\d+(?:-L\d+)?$`)

func extractCitations(text string, families []string) []Citation {
	citations := []Citation{}
	if len(families) == 0 {
		return citations
	}
	escaped := make([]string, len(families))
	for i, family := range families {
		escaped[i] = regexp.QuoteMeta(family)
	}
	token := regexp.MustCompile("(?s)\uE200(" + strings.Join(escaped, "|") + ")\uE202(.*?)\uE201")
	for _, match := range token.FindAllStringSubmatchIndex(text, -1) {
		parts := []string{}
		for _, part := range strings.Split(text[match[4]:match[5]], "\uE202") {
			if part = strings.TrimSpace(part); part != "" {
				parts = append(parts, part)
			}
		}
		locator := ""
		if len(parts) > 0 && lineLocator.MatchString(parts[len(parts)-1]) {
			locator = parts[len(parts)-1]
			parts = parts[:len(parts)-1]
		}
		valid := len(parts) > 0
		for _, part := range parts {
			if !sourceID.MatchString(part) {
				valid = false
				break
			}
		}
		if valid {
			citations = append(citations, Citation{
				text[match[0]:match[1]],
				text[match[2]:match[3]],
				parts,
				locator,
				match[0],
				match[1],
			})
		}
	}
	return citations
}
func stripCitations(text string, citations []Citation) string {
	sorted := append([]Citation(nil), citations...)
	sort.Slice(sorted, func(i, j int) bool {
		return sorted[i].Start > sorted[j].Start
	})
	for _, citation := range sorted {
		text = text[:citation.Start] + text[citation.End:]
	}
	return text
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49import java.util.*;
import java.util.regex.Pattern;

// Java string offsets are UTF-16 code units, matching substring().
record Citation(
    String raw, String family, List<String> sourceIds, String locator, int start, int end) {}

static final Pattern SOURCE_ID = Pattern.compile("^[A-Za-z0-9_-]+$");
static final Pattern LINE_LOCATOR = Pattern.compile("^L\\d+(?:-L\\d+)?$");

static List<Citation> extractCitations(String text, List<String> families) {
  var citations = new ArrayList<Citation>();
  if (families.isEmpty()) return citations;
  var familyPattern =
      families.stream().map(Pattern::quote).collect(java.util.stream.Collectors.joining("|"));
  var matcher =
      Pattern.compile("\uE200(" + familyPattern + ")\uE202(.*?)\uE201", Pattern.DOTALL)
          .matcher(text);
  while (matcher.find()) {
    var parts =
        new ArrayList<>(
            Arrays.stream(matcher.group(2).split("\uE202"))
                .map(String::strip)
                .filter(s -> !s.isEmpty())
                .toList());
    String locator = null;
    if (!parts.isEmpty() && LINE_LOCATOR.matcher(parts.get(parts.size() - 1)).matches())
      locator = parts.remove(parts.size() - 1);
    if (parts.isEmpty() || parts.stream().anyMatch(s -> !SOURCE_ID.matcher(s).matches()))
      continue;
    citations.add(
        new Citation(
            matcher.group(),
            matcher.group(1),
            List.copyOf(parts),
            locator,
            matcher.start(),
            matcher.end()));
  }
  return citations;
}

static String stripCitations(String text, List<Citation> citations) {
  var sorted = new ArrayList<>(citations);
  sorted.sort(Comparator.comparingInt(Citation::start).reversed());
  for (var citation : sorted)
    text = text.substring(0, citation.start()) + text.substring(citation.end());
  return text;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47CITATION_START = "\u{E200}"
CITATION_DELIMITER = "\u{E202}"
CITATION_STOP = "\u{E201}"

SOURCE_ID_RE = /\A[A-Za-z0-9_-]+\z/
LINE_LOCATOR_RE = /\AL\d+(?:-L\d+)?\z/

def extract_citations(text, families: ["cite"])
  return [] if families.empty?

  family_pattern = families.map { |family| Regexp.escape(family) }.join("|")
  token_re = Regexp.new(
    "#{Regexp.escape(CITATION_START)}" \
      "(?<family>#{family_pattern})" \
      "#{Regexp.escape(CITATION_DELIMITER)}" \
      "(?<body>.*?)" \
      "#{Regexp.escape(CITATION_STOP)}",
    Regexp::MULTILINE
  )

  text.enum_for(:scan, token_re).map do
    match = Regexp.last_match
    next unless match

    body = match[:body]
    next unless body

    parts = body.split(CITATION_DELIMITER).map(&:strip).reject(&:empty?)
    locator = parts.pop if parts.last&.match?(LINE_LOCATOR_RE)
    next if parts.empty? || parts.any? { |part| !part.match?(SOURCE_ID_RE) }

    {
      raw: match[0],
      family: match[:family],
      source_ids: parts,
      locator: locator,
      start: match.begin(0),
      end: match.end(0)
    }
  end.compact
end

def strip_citations(text, citations)
  citations.sort_by { |citation| -citation.fetch(:start) }.each_with_object(text.dup) do |citation, clean|
    clean[citation.fetch(:start)...citation.fetch(:end)] = ""
  end
end