Skip to content

implemented matching model and Added slack models #913

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
3 changes: 3 additions & 0 deletions backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ collect-static:
django-shell:
@CMD="python manage.py shell" $(MAKE) exec-backend-command-it

match-user:
@CMD="python manage.py matching_users $(model)" $(MAKE) exec-backend-command-it

dump-data:
@echo "Dumping Nest data"
@CMD="python manage.py dumpdata github owasp --indent=2" $(MAKE) exec-backend-command > backend/data/nest.json
Expand Down
129 changes: 129 additions & 0 deletions backend/apps/common/management/commands/matching_users.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""A command to perform fuzzy and exact matching of leaders with GitHub users models."""

from django.core.management.base import BaseCommand
from django.db.utils import DatabaseError
from thefuzz import fuzz

from apps.github.models.user import User
from apps.owasp.models.chapter import Chapter
from apps.owasp.models.committee import Committee
from apps.owasp.models.project import Project

MIN_NO_OF_WORDS = 2


class Command(BaseCommand):
help = "Process raw leaders for multiple models and suggest leaders."

def add_arguments(self, parser):
parser.add_argument(
"model_name",
type=str,
choices=["chapter", "committee", "project"],
help="Model name to process leaders for (chapter, committee, project)",
)
parser.add_argument(
"--threshold",
type=int,
default=85,
help="Threshold for fuzzy matching (0-100)",
)

def handle(self, *args, **kwargs):
model_name = kwargs["model_name"].lower()
threshold = max(0, min(kwargs["threshold"], 100))

model_map = {
"chapter": Chapter,
"committee": Committee,
"project": Project,
}

model_class = model_map.get(model_name)
if not model_class:
self.stdout.write(
self.style.ERROR("Invalid model name! Choose from: chapter, committee, project")
)
return

# Pre-fetch users
all_users = User.objects.values("id", "login", "name")
filtered_users = {
u["id"]: u for u in all_users if self._is_valid_user(u["login"], u["name"])
}

instances = model_class.objects.prefetch_related("suggested_leaders")
for instance in instances:
self.stdout.write(f"Processing leaders for {model_name.capitalize()} {instance.id}...")
exact_matches, fuzzy_matches, unmatched = self.process_leaders(
instance.leaders_raw, threshold, filtered_users
)

suggested_leader_ids = {user["id"] for user in exact_matches + fuzzy_matches}
instance.suggested_leaders.set(suggested_leader_ids)

if unmatched:
self.stdout.write(f"Unmatched leaders for {instance.name}: {unmatched}")

def _is_valid_user(self, login, name):
"""Check if user meets minimum requirements."""
return len(login) >= MIN_NO_OF_WORDS and name and len(name) >= MIN_NO_OF_WORDS

def process_leaders(self, leaders_raw, threshold, filtered_users):
"""Process leaders with optimized matching."""
if not leaders_raw:
return [], [], []

exact_matches = []
fuzzy_matches = []
unmatched_leaders = []
processed_leaders = set()

user_list = list(filtered_users.values())

for leader in leaders_raw:
if not leader or leader in processed_leaders:
continue

processed_leaders.add(leader)
leader_lower = leader.lower()

try:
exact_match = next(
(
u
for u in user_list
if u["login"].lower() == leader_lower
or (u["name"] and u["name"].lower() == leader_lower)
),
None,
)

if exact_match:
exact_matches.append(exact_match)
self.stdout.write(f"Exact match found for {leader}: {exact_match['login']}")
continue

matches = [
u
for u in user_list
if (fuzz.partial_ratio(leader_lower, u["login"].lower()) >= threshold)
or (
u["name"]
and fuzz.partial_ratio(leader_lower, u["name"].lower()) >= threshold
)
]

new_fuzzy_matches = [m for m in matches if m not in exact_matches]
if new_fuzzy_matches:
fuzzy_matches.extend(new_fuzzy_matches)
for match in new_fuzzy_matches:
self.stdout.write(f"Fuzzy match found for {leader}: {match['login']}")
else:
unmatched_leaders.append(leader)

except DatabaseError as e:
unmatched_leaders.append(leader)
self.stdout.write(self.style.ERROR(f"Error processing leader {leader}: {e}"))

return exact_matches, fuzzy_matches, unmatched_leaders
32 changes: 26 additions & 6 deletions backend/apps/owasp/admin.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""OWASP app admin."""

from django.contrib import admin
from django.contrib import admin, messages
from django.utils.safestring import mark_safe

from apps.owasp.models.chapter import Chapter
Expand Down Expand Up @@ -41,12 +41,31 @@ def custom_field_owasp_url(self, obj):
f"<a href='https://owasp.org/{obj.key}' target='_blank'>↗️</a>"
)

def approve_suggested_leaders(self, request, queryset):
"""Approve all suggested leaders for selected entities."""
for entity in queryset:
suggestions = entity.suggested_leaders.all()
entity.leaders.add(*suggestions)
self.message_user(
request,
f"Approved {suggestions.count()} leader suggestions for {entity.name}",
messages.SUCCESS,
)

custom_field_github_urls.short_description = "GitHub 🔗"
custom_field_owasp_url.short_description = "OWASP 🔗"
approve_suggested_leaders.short_description = "Approve all suggested leaders"


class LeaderEntityAdmin(admin.ModelAdmin, GenericEntityAdminMixin):
"""Admin class for entities that have leaders."""

actions = ["approve_suggested_leaders"]
filter_horizontal = ("suggested_leaders",)


class ChapterAdmin(admin.ModelAdmin, GenericEntityAdminMixin):
autocomplete_fields = ("owasp_repository",)
class ChapterAdmin(LeaderEntityAdmin):
autocomplete_fields = ("owasp_repository", "leaders")
list_display = (
"name",
"region",
Expand All @@ -62,8 +81,8 @@ class ChapterAdmin(admin.ModelAdmin, GenericEntityAdminMixin):
search_fields = ("name", "key")


class CommitteeAdmin(admin.ModelAdmin):
autocomplete_fields = ("owasp_repository",)
class CommitteeAdmin(LeaderEntityAdmin):
autocomplete_fields = ("owasp_repository", "leaders")
search_fields = ("name",)


Expand Down Expand Up @@ -92,12 +111,13 @@ class PostAdmin(admin.ModelAdmin):
)


class ProjectAdmin(admin.ModelAdmin, GenericEntityAdminMixin):
class ProjectAdmin(LeaderEntityAdmin):
autocomplete_fields = (
"organizations",
"owasp_repository",
"owners",
"repositories",
"leaders",
)
list_display = (
"custom_field_name",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# Generated by Django 5.1.6 on 2025-02-22 21:43

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("github", "0015_alter_release_author"),
("owasp", "0014_project_custom_tags"),
]

operations = [
migrations.AddField(
model_name="chapter",
name="leaders",
field=models.ManyToManyField(
blank=True,
related_name="normal_%(class)s",
to="github.user",
verbose_name="Leaders",
),
),
migrations.AddField(
model_name="chapter",
name="suggested_leaders",
field=models.ManyToManyField(
blank=True,
related_name="exact_matched_%(class)s",
to="github.user",
verbose_name="Exact Match Users",
),
),
migrations.AddField(
model_name="committee",
name="leaders",
field=models.ManyToManyField(
blank=True,
related_name="normal_%(class)s",
to="github.user",
verbose_name="Leaders",
),
),
migrations.AddField(
model_name="committee",
name="suggested_leaders",
field=models.ManyToManyField(
blank=True,
related_name="exact_matched_%(class)s",
to="github.user",
verbose_name="Exact Match Users",
),
),
migrations.AddField(
model_name="project",
name="leaders",
field=models.ManyToManyField(
blank=True,
related_name="normal_%(class)s",
to="github.user",
verbose_name="Leaders",
),
),
migrations.AddField(
model_name="project",
name="suggested_leaders",
field=models.ManyToManyField(
blank=True,
related_name="exact_matched_%(class)s",
to="github.user",
verbose_name="Exact Match Users",
),
),
]
12 changes: 12 additions & 0 deletions backend/apps/owasp/migrations/0031_merge_20250320_2001.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Generated by Django 5.1.7 on 2025-03-20 20:01

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("owasp", "0015_chapter_leaders_chapter_suggested_leaders_and_more"),
("owasp", "0030_chapter_is_leaders_policy_compliant_and_more"),
]

operations = []
11 changes: 11 additions & 0 deletions backend/apps/owasp/models/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,17 @@ def get_metadata(self):
extra={"repository": getattr(self.owasp_repository, "name", None)},
)

# M2M
suggested_leaders = models.ManyToManyField(
"github.User",
verbose_name="Exact Match Users",
related_name="exact_matched_%(class)s",
blank=True,
)
leaders = models.ManyToManyField(
"github.User", verbose_name="Leaders", related_name="normal_%(class)s", blank=True
)

def get_related_url(self, url, exclude_domains=(), include_domains=()):
"""Get OWASP entity related URL."""
if (
Expand Down
Loading