# create  lists to store NER tags and tokens for each sentence.
all_ner_tags, all_ner_tokens = [], []
#o_tag: This is the "Outside" tag (O), used to mark tokens that are not part of any entity.
o_tag = "O"

#Processing Each Record in the Annotated Data
for rec in ner_annotated_data:
    ner_tags, ner_tokens = [], []

    #Extract the sentence (text) from the record.
    text = rec["data"]["text"]
    #Skipping Records Without Valid Annotations:
    #This checks if there are no annotations in the current record or if the annotation result is missing or empty. If any of these conditions are true, the code skips this record and moves to the next one.
    if len(rec["annotations"]) == 0 or "result" not in rec["annotations"][0] or len(rec["annotations"][0]["result"]) == 0:
        continue

    # sort rec["annotations"][0]["result"] based on start index
    rec["annotations"][0]["result"].sort(key=lambda x: x["value"]["start"])
    # x["value"]["start"]: Refers to the starting position of the entity in the text.

    # collect all ranges with their labels
    ranges = []
    for r in rec["annotations"][0]["result"]:
        ranges.append((range(r["value"]["start"], r["value"]["end"]+1), r["value"]["labels"][0]))

    # split text into tokens
    tokens = text.split()
    token_ranges = []
    c = 0 #keeps track of the character position of each token
    for i, token in enumerate(tokens):
        token_ranges.append( (range(c, c+len(token)), token) )
        c += len(token) + 1

    # find all tokens that are in the ranges
    for token_range in token_ranges:
        #Before checking if a token matches an entity, the flag is_found is set to False. This means that, by default, we assume the token does not belong to any entity.
        is_found = False

        for sub_range in ranges:
            if all(e in sub_range[0] for e in token_range[0]):
                ner_tags.append(sub_range[1])
                ner_tokens.append(token_range[1])
                is_found = True
                break

        # If a token is not found in any entity range after the inner loop,
        #this means that the token does not correspond to any named entity.
        if not is_found:
            ner_tags.append(o_tag)
            ner_tokens.append(token_range[1])

    # format BI prefix
    for i, tag in enumerate(ner_tags):
        #processing the first token
        if i == 0 and ner_tags[i] != o_tag:
            ner_tags[i] = f"B-{ner_tags[i]}"
            continue

        # skipping tokens labeled as O
        if i == 0 or ner_tags[i] == o_tag:
            continue

        #handling sub sequent tokens
        if ner_tags[i-1].replace("B-","").replace("I-","") == ner_tags[i]:
            ner_tags[i] = f"I-{ner_tags[i]}"
        else:
            ner_tags[i] = f"B-{ner_tags[i]}"


    all_ner_tags.append(ner_tags)
    all_ner_tokens.append(ner_tokens)