Skip to content

Commit b5cf5aa

Browse files
regulartimregulartim
authored
* Include mass scanners in advanced API by default. Closes GreedyBear-Project#580 (GreedyBear-Project#581) * Partly revert "added mass scanner exclusion as default" This reverts commit f953887. * adapt tests * add "tor exit nodes" to default excludes * add test case for tor exit node inclusion * add test case for tor exit node inclusion (ii) * fix syntax * rename method * Upgrade Django to 5.2. Closes GreedyBear-Project#502 (GreedyBear-Project#579) * bump django-rest-email-auth * bump django to 5.2 * bump postgres to 18 (this requires manual manual intervention when upgrading GreedyBear) * Link to admin interface for staff users. Closes GreedyBear-Project#529 (GreedyBear-Project#582) * remove restriction to only show link to superusers * fix indentation * bump 2.0.0 * adapt CI
1 parent d0315b1 commit b5cf5aa

11 files changed

Lines changed: 70 additions & 33 deletions

File tree

‎.env_template‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ COMPOSE_FILE=docker/default.yml:docker/local.override.yml
1313
#COMPOSE_FILE=docker/default.yml:docker/local.override.yml:docker/elasticsearch.yml
1414

1515
# If you want to run a specific version, populate this
16-
# REACT_APP_INTELOWL_VERSION="1.6.8"
16+
# REACT_APP_INTELOWL_VERSION="2.0.0"

‎.github/workflows/pull_request_automation.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070
postgres_db: greedybear_db
7171
postgres_user: user
7272
postgres_password: password
73-
postgres_version: 13
73+
postgres_version: 18
7474
use_memcached: false
7575
use_elastic_search: false
7676
use_rabbitmq: true

‎api/views/feeds.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@ def feeds(request, feed_type, attack_type, prioritize, format_):
2626
prioritize (str): Prioritization mechanism to use (e.g., recent, persistent).
2727
format_ (str): Desired format of the response (e.g., json, csv, txt).
2828
include_mass_scanners (bool): query parameter flag to include IOCs that are known mass scanners.
29+
include_tor_exit_nodes (bool): query parameter flag to include IOCs that are known tor exit nodes.
2930
3031
Returns:
3132
Response: The HTTP response with formatted IOC data.
3233
"""
3334
logger.info(f"request /api/feeds with params: feed type: {feed_type}, " f"attack_type: {attack_type}, prioritization: {prioritize}, format: {format_}")
3435

3536
feed_params = FeedRequestParams({"feed_type": feed_type, "attack_type": attack_type, "format_": format_})
37+
feed_params.apply_default_filters(request.query_params)
3638
feed_params.set_prioritization(prioritize)
37-
if request.query_params and "include_mass_scanners" in request.query_params:
38-
feed_params.include_mass_scanners()
3939

4040
valid_feed_types = get_valid_feed_types()
4141
iocs_queryset = get_queryset(request, feed_params, valid_feed_types)
@@ -58,9 +58,8 @@ def feeds_pagination(request):
5858

5959
feed_params = FeedRequestParams(request.query_params)
6060
feed_params.format = "json"
61+
feed_params.apply_default_filters(request.query_params)
6162
feed_params.set_prioritization(request.query_params.get("prioritize"))
62-
if request.query_params and "include_mass_scanners" in request.query_params:
63-
feed_params.include_mass_scanners()
6463

6564
valid_feed_types = get_valid_feed_types()
6665
iocs_queryset = get_queryset(request, feed_params, valid_feed_types)
@@ -83,8 +82,8 @@ def feeds_advanced(request):
8382
attack_type (str): Type of attack to filter. (supported: `scanner`, `payload_request`, `all`; default: `all`)
8483
max_age (int): Maximum number of days since last occurrence. E.g. an IOC that was last seen 4 days ago is excluded by default. (default: 3)
8584
min_days_seen (int): Minimum number of days on which an IOC must have been seen. (default: 1)
86-
include_reputation (str): `;`-separated list of reputation values to include, e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all) this has precedence over exclusion
87-
exclude_reputation (str): `;`-separated list of reputation values to exclude, e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude mass scanners)
85+
include_reputation (str): `;`-separated list of reputation values to include, e.g. `known attacker` or `known attacker;` to include IOCs without reputation. (default: include all)
86+
exclude_reputation (str): `;`-separated list of reputation values to exclude, e.g. `mass scanner` or `mass scanner;bot, crawler`. (default: exclude none)
8887
feed_size (int): Number of IOC items to return. (default: 5000)
8988
ordering (str): Field to order results by, with optional `-` prefix for descending. (default: `-last_seen`)
9089
verbose (bool): `true` to include IOC properties that contain a lot of data, e.g. the list of days it was seen. (default: `false`)

‎api/views/utils.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,14 @@ def __init__(self, query_params: dict):
7575
self.paginate = query_params.get("paginate", "false").lower()
7676
self.format = query_params.get("format_", "json").lower()
7777
self.feed_type_sorting = None
78-
self.exclude_reputation.append("mass scanner")
7978

80-
def include_mass_scanners(self):
81-
self.exclude_reputation.remove("mass scanner")
79+
def apply_default_filters(self, query_params):
80+
if not query_params:
81+
query_params = dict()
82+
if "include_mass_scanners" not in query_params:
83+
self.exclude_reputation.append("mass scanner")
84+
if "include_tor_exit_nodes" not in query_params:
85+
self.exclude_reputation.append("tor exit node")
8286

8387
def set_prioritization(self, prioritize: str):
8488
match prioritize:
@@ -90,7 +94,7 @@ def set_prioritization(self, prioritize: str):
9094
self.ordering = "-last_seen"
9195
case "persistent":
9296
self.max_age = "14"
93-
self.min_days_seen: "10"
97+
self.min_days_seen = "10"
9498
if "feed_type" in self.ordering:
9599
self.feed_type_sorting = self.ordering
96100
self.ordering = "-attack_count"
@@ -155,14 +159,11 @@ def get_queryset(request, feed_params, valid_feed_types):
155159
query_dict["number_of_days_seen__gte"] = int(feed_params.min_days_seen)
156160
if feed_params.include_reputation:
157161
query_dict["ip_reputation__in"] = feed_params.include_reputation
158-
for reputation_type in feed_params.include_reputation:
159-
if reputation_type in feed_params.exclude_reputation:
160-
feed_params.exclude_reputation.remove(reputation_type)
161162

162163
iocs = (
163164
IOC.objects.filter(**query_dict)
164165
.filter(Q(cowrie=True) | Q(log4j=True) | Q(general_honeypot__active=True))
165-
.exclude(Q() if "nothing" in feed_params.exclude_reputation else Q(ip_reputation__in=feed_params.exclude_reputation))
166+
.exclude(ip_reputation__in=feed_params.exclude_reputation)
166167
.annotate(value=F("name"))
167168
.annotate(honeypots=ArrayAgg("general_honeypot__name"))
168169
.order_by(feed_params.ordering)[: int(feed_params.feed_size)]

‎docker/.version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
REACT_APP_GREEDYBEAR_VERSION="1.6.8"
1+
REACT_APP_GREEDYBEAR_VERSION="2.0.0"

‎docker/default.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ x-no-healthcheck: &no-healthcheck
44

55
services:
66
postgres:
7-
image: library/postgres:13-alpine
7+
image: library/postgres:18-alpine
88
container_name: greedybear_postgres
99
volumes:
1010
- postgres_data:/var/lib/postgresql/data/

‎frontend/src/layouts/widget/UserMenu.jsx‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,9 @@ function UserMenu(props) {
2929
</DropdownItem>
3030
<DropdownItem divider />
3131
{/* Django Admin Interface */}
32-
{isSuperuser && (
33-
<DropdownNavLink to="/admin/" target="_blank">
34-
<IoMdSettings className="me-2" /> Django Admin Interface
35-
</DropdownNavLink>
36-
)}
32+
<DropdownNavLink to="/admin/" target="_blank">
33+
<IoMdSettings className="me-2" /> Django Admin Interface
34+
</DropdownNavLink>
3735
{/* API Access/Sessions */}
3836
<DropdownNavLink to="/me/sessions">
3937
<IoMdKey className="me-2" /> API Access / Sessions

‎requirements/project-requirements.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ celery==5.5.3
33
# if you change this, update the documentation
44
elasticsearch-dsl==8.18.0
55

6-
Django==4.2.24
6+
Django==5.2.7
77
djangorestframework==3.16.1
8-
django-rest-email-auth==4.0.3
8+
django-rest-email-auth==5.0.0
99
django-ses==4.4.0
1010

1111
psycopg2-binary==2.9.10

‎tests/__init__.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,28 @@ def setUpTestData(cls):
5959
expected_interactions=11.1,
6060
)
6161

62+
cls.ioc_3 = IOC.objects.create(
63+
name="100.100.100.100",
64+
type=iocType.IP.value,
65+
first_seen=cls.current_time,
66+
last_seen=cls.current_time,
67+
days_seen=[cls.current_time],
68+
number_of_days_seen=1,
69+
attack_count=1,
70+
interaction_count=1,
71+
log4j=False,
72+
cowrie=True,
73+
scanner=True,
74+
payload_request=True,
75+
related_urls=[],
76+
ip_reputation="tor exit node",
77+
asn="12345",
78+
destination_ports=[22, 23, 24],
79+
login_attempts=1,
80+
recurrence_probability=0.1,
81+
expected_interactions=11.1,
82+
)
83+
6284
cls.ioc.general_honeypot.add(cls.heralding) # FEEDS
6385
cls.ioc.general_honeypot.add(cls.ciscoasa) # FEEDS
6486
cls.ioc.save()

‎tests/test_scoring_utils.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ class TestFeatExtraction(CustomTestCase):
7878
def test_data_retrieval(self):
7979
"""Test with sample IoCs"""
8080
data = get_current_data()
81-
self.assertEqual(len(data), 2)
81+
self.assertEqual(len(data), 3)
8282

8383
def test_feature_extraction(self):
8484
"""Test with sample IoCs"""
@@ -92,8 +92,9 @@ def test_feature_extraction(self):
9292
self.assertEqual(len(feature["days_seen"]), 1)
9393
self.assertEqual(str(feature["days_seen"][0]), today)
9494
self.assertEqual(feature["asn"], "12345")
95-
self.assertEqual(set(feature["honeypots"]), set(["heralding", "ciscoasa", "log4j", "cowrie"]))
96-
self.assertEqual(feature["honeypot_count"], 4)
95+
self.assertTrue(len(feature["honeypots"]) > 0)
96+
self.assertTrue(set(feature["honeypots"]).issubset({"heralding", "ciscoasa", "log4j", "cowrie"}))
97+
self.assertEqual(feature["honeypot_count"], len(feature["honeypots"]))
9798
self.assertEqual(feature["destination_port_count"], 3)
9899
self.assertEqual(feature["days_seen_count"], 1)
99100
self.assertEqual(feature["active_timespan"], 1)

0 commit comments

Comments
 (0)