Merge branch 'SBO' into 'dev'

Image Analyser

See merge request sboettger/foto-puzzler!3

.gitignore

@@ -4,4 +4,6 @@ __pycache__/*
 
 # image files
 /resources/*
+!/resources/demo_images/
+!/resources/test data/
 !/resources/README.md

README.md

-# Foto Puzzler
-
-# Features
-Goal of this project is to create a software, that can solve a puzzle based on a picture of all pieces 
+# Mario Jigsaw Genius
+## Features
+ - Extract puzzle pieces from a photo
+ - Analyze a puzzle piece and convert them into a puzzle piece object
+ - Solve the puzzle
+ - Reassemble the puzzle pieces correctly
 
 ## Target
 The project aims to implement an algorithm that makes it possible to complete a puzzle using a photo. To do this, the patterns and shapes of the puzzle pieces must be matched and placed together. In order to save unnecessary additional work in the post-processing of the puzzle pieces, an automatically processed photo should be used, which already contains brightness and distortion corrections (e.g. smartphone camera recording).
 
+## Project structure
+The project ist structured in three parts:
+ - Image Processing
+    - Extract puzzle pieces from photo
+    - Reassemble the puzzle pieces
+ - Image Analyzer
+    - Analyze a puzzle piece and convert them into a puzzle piece object
+ - Puzzle Solving
+    - Solve the puzzle
+
 ## Project status
-Work in progress
+All three parts working independently with test data.
+The main script MarioJigsawGenius.py is currently not working
 
 ## Author
 Jan Genders, Jessica Dreyer, Sebastian Böttger
 
 ## License
-This project is licensed under the MIT License - see the LICENSE file for details.
\ No newline at end of file
+This project is licensed under the GNU GPL V3 - see the LICENSE file for details.
\ No newline at end of file

doc/diagrams/class_diagram.uxf

-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<diagram program="umlet" version="15.0.0">
-  <help_text>Space for diagram notes</help_text>
-  <zoom_level>9</zoom_level>
-  <element>
-    <id>UMLClass</id>
-    <coordinates>
-      <x>108</x>
-      <y>45</y>
-      <w>315</w>
-      <h>153</h>
-    </coordinates>
-    <panel_attributes>Puzzle
+<diagram program="umletino" version="15.1"><zoom_level>9</zoom_level><help_text>Space for diagram notes</help_text><element><id>UMLClass</id><coordinates><x>180</x><y>42</y><w>315</w><h>153</h></coordinates><panel_attributes>Puzzle
 --
 - size : int[ ]
 - width : int
@@ -20,18 +8,7 @@
 + set_piece(piece : PuzzlePiece, idx : int)
 + get_edge_pieces(is_sorted : bool) : PuzzlePiece[ ]
 + get_inner_pieces(is_sorted : bool) : PuzzlePiece[ ]
-+ get_corners(is_sorted : bool) : PuzzlePiece[ ]</panel_attributes>
-    <additional_attributes/>
-  </element>
-  <element>
-    <id>UMLClass</id>
-    <coordinates>
-      <x>99</x>
-      <y>270</y>
-      <w>324</w>
-      <h>162</h>
-    </coordinates>
-    <panel_attributes>PuzzlePiece
++ get_corners(is_sorted : bool) : PuzzlePiece[ ]</panel_attributes><additional_attributes></additional_attributes></element><element><id>UMLClass</id><coordinates><x>171</x><y>267</y><w>324</w><h>162</h></coordinates><panel_attributes>PuzzlePiece
 --
 - id : int
 - category : Enum = {"EDGE", "CORNER", "INNER"}
@@ -43,105 +20,90 @@
 - image_path : String
 --
 + rotate(top_side : int) : None
-</panel_attributes>
-    <additional_attributes/>
-  </element>
-  <element>
-    <id>Relation</id>
-    <coordinates>
-      <x>252</x>
-      <y>189</y>
-      <w>36</w>
-      <h>99</h>
-    </coordinates>
-    <panel_attributes>lt=&lt;&lt;&lt;&lt;-
+</panel_attributes><additional_attributes></additional_attributes></element><element><id>Relation</id><coordinates><x>324</x><y>186</y><w>36</w><h>99</h></coordinates><panel_attributes>lt=&lt;&lt;&lt;&lt;-
 m1=1
-m2=*</panel_attributes>
-    <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
-  </element>
-  <element>
-    <id>UMLClass</id>
-    <coordinates>
-      <x>126</x>
-      <y>495</y>
-      <w>270</w>
-      <h>117</h>
-    </coordinates>
-    <panel_attributes>Side
+m2=*</panel_attributes><additional_attributes>10;10;10;90</additional_attributes></element><element><id>UMLClass</id><coordinates><x>198</x><y>492</y><w>270</w><h>117</h></coordinates><panel_attributes>Side
 --
 - id : int
 - side_type : Enum {"FLAT", "HOLE", "HEAD"}
 - outline : np.array
 - width : int
 - height : int
-- is_sorted : boolean</panel_attributes>
-    <additional_attributes/>
-  </element>
-  <element>
-    <id>Relation</id>
-    <coordinates>
-      <x>252</x>
-      <y>423</y>
-      <w>36</w>
-      <h>90</h>
-    </coordinates>
-    <panel_attributes>lt=&lt;&lt;&lt;&lt;&lt;-
+- is_sorted : boolean</panel_attributes><additional_attributes></additional_attributes></element><element><id>Relation</id><coordinates><x>324</x><y>420</y><w>36</w><h>90</h></coordinates><panel_attributes>lt=&lt;&lt;&lt;&lt;&lt;-
 m1=1
-m2=4</panel_attributes>
-    <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
-  </element>
-  <element>
-    <id>UMLClass</id>
-    <coordinates>
-      <x>504</x>
-      <y>414</y>
-      <w>153</w>
-      <h>72</h>
-    </coordinates>
-    <panel_attributes>MatchingCounterpiece
+m2=4</panel_attributes><additional_attributes>10;10;10;80</additional_attributes></element><element><id>UMLClass</id><coordinates><x>0</x><y>537</y><w>153</w><h>72</h></coordinates><panel_attributes>MatchingCounterpiece
 --
 - piece : PuzzlePiece
 - side : Side
-- distance : int</panel_attributes>
-    <additional_attributes/>
-  </element>
-  <element>
-    <id>Relation</id>
-    <coordinates>
-      <x>414</x>
-      <y>342</y>
-      <w>108</w>
-      <h>117</h>
-    </coordinates>
-    <panel_attributes>lt=-</panel_attributes>
-    <additional_attributes>10.0;10.0;100.0;110.0</additional_attributes>
-  </element>
-  <element>
-    <id>Relation</id>
-    <coordinates>
-      <x>387</x>
-      <y>459</y>
-      <w>135</w>
-      <h>135</h>
-    </coordinates>
-    <panel_attributes>lt=-</panel_attributes>
-    <additional_attributes>130.0;10.0;10.0;130.0</additional_attributes>
-  </element>
-  <element>
-    <id>UMLClass</id>
-    <coordinates>
-      <x>468</x>
-      <y>45</y>
-      <w>792</w>
-      <h>90</h>
-    </coordinates>
-    <panel_attributes>PuzzleSolver
+- distance : int</panel_attributes><additional_attributes></additional_attributes></element><element><id>Relation</id><coordinates><x>126</x><y>339</y><w>63</w><h>216</h></coordinates><panel_attributes>lt=-</panel_attributes><additional_attributes>50;10;10;10;10;220</additional_attributes></element><element><id>Relation</id><coordinates><x>144</x><y>573</y><w>72</w><h>27</h></coordinates><panel_attributes>lt=-</panel_attributes><additional_attributes>10;10;60;10</additional_attributes></element><element><id>UMLClass</id><coordinates><x>522</x><y>42</y><w>684</w><h>90</h></coordinates><panel_attributes>PuzzleSolver
 --
 --
 _+ get_compatible_side_type(side_type : SideType) : SideType_
 _+ get_possible_sides(piece : PuzzlePiece, current_side : Side, idx_possible_side : int) : Side[ ]_
 _+ find_compatible(current_piece : PuzzlePiece, current_side : Side, possible_pieces : PuzzlePiece[ ], idx_help : int) : (PuzzlePiece, Side)_
-_+ solve(puzzle : Puzzle) : Puzzle_</panel_attributes>
-    <additional_attributes/>
-  </element>
-</diagram>
+_+ solve(puzzle : Puzzle) : Puzzle_</panel_attributes><additional_attributes></additional_attributes></element><element><id>UMLClass</id><coordinates><x>522</x><y>159</y><w>540</w><h>450</h></coordinates><panel_attributes>ImageAnalyser
+--
+- image : np.ndarray
+- piece_id : int
+- high_spot_search_size : int
+- angle_base_size : int
+- corner_value : float
+- max_hight_flat_side : int
+- puzzle_piece : PuzzlePiece
+--
++ __init__(
+    image : np.ndarray = np.zeros((10,10)), 
+    piece_id : int = 0, high_spot_search_size : int = 50,
+    angle_base_size : int = 40, corner_value : float = 0.03,
+    max_hight_flat_side : int = 20)
+
++ analyse() : None
++ create_outline_image(image : np.ndarray) : None
++ convert_to_cart_path(outline_image : np.ndarray) : None
++ sort_by_diff(cart_path : np.ndarray) : np.ndarray
++ convert_to_pol_path(cart_path : np.ndarray) : None
++ get_center_point(cart_path : np.ndarray) : np.ndarray
++ find_high_spots_index(pol_path : np.ndarray, size : int = 20) : None
++ calculate_angle(index_high_spots : np.ndarray, size : int = 20) : None
++ calculate_gamma(points : np.ndarray) : np.ndarray
++ extract_corners(angles : np.ndarray, index_high_spots : np.ndarray, corner_value : float = 0.04) : None
++ split_sites(cart_path : np.ndarray, index_corners : np.ndarray) : None
++ normalize_sides() : None
++ calculate_straight_angle(side_path : np.ndarray) : None
++ rotate_side(side_path : np.ndarray, angle : float) : np.ndarray
++ polish_side_paths() : None
++ convert_to_binary_array(side_path : np.ndarray) : np.ndarray
++ create_puzzle_piece(sides_paths : List[np.ndarray]) : PuzzlePiece
++ determine_side_type(side_path : np.ndarray) : SideType
+</panel_attributes><additional_attributes></additional_attributes></element><element><id>UMLClass</id><coordinates><x>1089</x><y>159</y><w>306</w><h>135</h></coordinates><panel_attributes>PuzzleConnector
+--
+- rows : int
+- columns : int
+- puzzle_pieces : List
+--
++ __init__(rows : int, columns : int)
++ add_piece(piece : PuzzlePiece) : None
++ compose() : np.ndarray
++ rotate_pieces(puzzle : Puzzle) : None
+</panel_attributes><additional_attributes></additional_attributes></element><element><id>UMLClass</id><coordinates><x>1089</x><y>312</y><w>306</w><h>90</h></coordinates><panel_attributes>PuzzleConnectorWithAnimation
+--
+- frames : List
+--
++ __init__(rows : int, columns : int)
++ compose_with_animation() : None
++ animate_puzzle() : None</panel_attributes><additional_attributes></additional_attributes></element><element><id>UMLClass</id><coordinates><x>1089</x><y>411</y><w>594</w><h>198</h></coordinates><panel_attributes>PuzzlePieceExtractor
+--
+- threshold : int
+- minimum_res : int
+- clicked_point_background : Tuple
+- clicked_point_shadow : Tuple
+--
++ __init__(threshold : int, minimum_res : int)
++ get_color(event : cv2.EVENT, x : int, y : int, flags : Any, param : Any) : None
++ select_colors(image : np.ndarray) : Tuple
++ remove_background(image : np.ndarray, background_color : np.ndarray, shadow_color : np.ndarray) : np.ndarray
++ erode_n_dilate(image : np.ndarray) : np.ndarray
++ splitting(image : np.ndarray) : List
++ remove_border_artifacts(image_array : np.ndarray) : np.ndarray
++ generate_pieces(image : np.ndarray) : Tuple
+</panel_attributes><additional_attributes></additional_attributes></element></diagram>
\ No newline at end of file

doc/notes/notes.txt

-- Fotos machen
-- Ecken vom Puzzleteil feststellen
-	-> Puzzleteil kategorisieren
-- Seiten Analysieren
-	-> Kontor erstelen
-	-> Breite feststellen
-
-# Phase 1
-- Einzelne Seiten vergleichen
-
-
-
-Aufgabenaufteilung:
-
-#1 Gesamtfoto in einzelbilder aufteilen
-	-> Am Ende auch wieder Zusammensetzten
-	
-#2 Kanten/Eckendektion
-	-> Kategorisieren
-	-> Outline erstelen (Array erstellen)
-
-#3 Puzzle lösen
-
-
-
-UML:
-
-__Puzzle__												# Puzzle
-- Größe [Breite, Höhe]									# size: [width, height]
-- Breite										# width
-- Höhe											# height
-- PuzzleTeile											# pieces: PuzzlePiece
-- 
-
-__Puzzelteil__											# PuzzlePiece
-- Id													# id	
-- Seiten[4]												# sides : Side
-- Kategorie {"Rand","Ecke", "Mitte"}					# category: Enum {"EDGE", "CORNER", "INNER"}
-- Anzahl Einbuchtungen									# number_of_holes
-- Anzahl Ausbuchtungen									# number_of_heads
-- Obere Seite											# top_side: int = 0
-- Bildpfad												# image_path: String
-- is_sorted
-
-__Seite__												# Side
-- Id													# id
-- Typ {"Flach", "Einbuchtung", "Ausbuchtung"}			# side_type: Enum {"FLAT", "HOLE", "HEAD"}
-- Kontur												# outline: np.array
-- Breite												# width: int
-- Höhe
-- is_sorted
\ No newline at end of file

src/puzzle solving/test_solving.py → resources/test data/testPuzzleData.py

-# %% Imports
+#%% imports
 import sys
 import numpy as np
 
-sys.path.append('../general') # adjusting the path for the following imports
+sys.path.append('../../src/general')
 
 from Puzzle import Puzzle
 from PuzzlePiece import PuzzlePiece
 from PuzzlePiece import Category
 from Side import Side
 from Side import SideType
-from PuzzleSolver import PuzzleSolver
-
-
-# %% test data 2x2-Puzzle
 
+#%% test data 2x2-puzzle
 def get_puzzle_2x2():
     
     piece1_side1_outline = np.array([   [0,0,0,1,1,1,0,0,0],
@@ -113,8 +110,6 @@ def get_puzzle_2x2():
     
     return puzzle
 
-# %% test data 3x3-Puzzle
-
 def get_puzzle_3x3():
     piece0_side0_outline = np.array([   [0,0,0,1,1,1,0,0,0],
                                         [0,0,1,0,0,0,1,0,0],
@@ -336,13 +331,3 @@ def get_puzzle_3x3():
     puzzle.set_pieces(np.array([piece0, piece1, piece2, piece3, piece4, piece5, piece6, piece7, piece8]))
     
     return puzzle
\ No newline at end of file
-
-#%% test
-
-puzzle = get_puzzle_3x3()
-
-sorted_puzzle = PuzzleSolver.solve(puzzle)
-        
-for p in sorted_puzzle.pieces:
-    print("top side piece ", p.id, " = ", p.top_side)
-        
\ No newline at end of file

src/MarioJigsawGenius.py

+# -*- coding: utf-8 -*-
+"""
+Created on Sun Sep  1 18:40:43 2024
+
+@author: Sebastian
+"""
+
+#%% imports
+import sys
+sys.path.append("./general/")
+sys.path.append("./image_processing/")
+sys.path.append("./image_analysis/")
+sys.path.append("./puzzle_solving/")
+import cv2
+import numpy as np
+from scipy.spatial import distance
+import matplotlib.pyplot as plt
+
+from PuzzlePieceExtractor import PuzzlePieceExtractor
+from ImageAnalyser import ImageAnalyser
+from Side import Side, SideType
+from PuzzlePiece import PuzzlePiece, Category
+
+#%% script
+# load image
+print("load image")
+image_path= "../resources/demo_images/all_mixed.jpg"
+image = cv2.imread(image_path)
+
+# create puzzle piece extractor
+print("create PuzzlePieceExtractor")
+extractor = PuzzlePieceExtractor()
+
+# extract puzzle pieces
+print("extract pieces")
+im_pieces, im_gray_pieces = extractor.generate_pieces(image)
+
+# convetr image to puzzle pieces
+print("convert image to piece objects")
+pieces = []
+image_analyser = ImageAnalyser()
+
+
+for i in range(len(im_gray_pieces)):
+    print(f"piece {i} of {len(im_gray_pieces)}")
+    image_analyser.image = im_gray_pieces[i]
+    image_analyser.piece_id = i
+    pieces.append(image_analyser.analyse())
+    
+
+for i in range(len(im_gray_pieces)):
+    cv2.imshow("Test", im_gray_pieces[i])
+    cv2.waitKey(0)
\ No newline at end of file