Update app.py

app.py

@@ -9,10 +9,41 @@ for idx, url in enumerate(urls):
   image = Image.open(requests.get(url, stream=True).raw)
   image.save(f"image_{idx}.png")
 
+from transformers import BertTokenizer, BasicTokenizer
+from transformers.tokenization_utils import _is_punctuation
 
-image_processor = AutoImageProcessor.from_pretrained("microsoft/swin-base-patch4-window7-224")
-tokenizer = tokenizer =BertTokenizerFast.from_pretrained("onlplab/alephbert-base")
-model = VisionEncoderDecoderModel.from_pretrained("sivan22/hdd-words-ocr")
+class OurBasicTokenizer(BasicTokenizer):
+    def _run_split_on_punc(self, text, never_split=None):
+        """Splits punctuation on a piece of text."""
+        if text in self.never_split or (never_split and text in never_split):
+            return [text]
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char) and char != "'" and not (char == '"' and i + 1 < len(chars) and not _is_punctuation(chars[i + 1])):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+
+def RabbinicTokenizer(tok):
+    tok.basic_tokenizer = OurBasicTokenizer(tok.basic_tokenizer.do_lower_case, tok.basic_tokenizer.never_split)
+    return tok
+
+
+image_processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
+tokenizer = RabbinicTokenizer(BertTokenizer.from_pretrained("sivan22/BEREL"))
+model = VisionEncoderDecoderModel.from_pretrained("sivan22/ABBA-HTR")
 
 
 def process_image(image):
@@ -28,7 +59,7 @@ def process_image(image):
     return generated_text
 
 title = "הדגמה: פענוח כתב יד באמצעות בינה מלאכותית"
-description = "על בסיס מודל swin בצד התמונה, ומודל alephbert בצד הטקסט."
+description = "על בסיס טכנולוגיית trOCR"
 article = "<p style='text-align: center'>sivan22</p>"
 examples =[["image_0.png"], ["image_1.png"], ["image_2.png"]]