From ec8ebf328066808233395c15750d72c860eb4d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Ha=C3=9Fel?= Date: Tue, 18 Jun 2024 10:24:39 +0200 Subject: [PATCH] Search.query + Offer.price --- .../kleinanzeigen/KleinanzeigenApi.java | 31 ++++++++++-- .../kleinanzeigen/offer/Offer.java | 16 ++++++ .../kleinanzeigen/offer/OfferCreate.java | 17 ++++++- .../kleinanzeigen/offer/OfferDto.java | 13 +++++ .../kleinanzeigen/search/Search.java | 49 +++++++++++++++++++ .../kleinanzeigen/search/SearchCreate.java | 31 ++++++++++++ .../kleinanzeigen/search/SearchDto.java | 48 ++++++++++++++++++ .../search/SearchRepository.java | 11 +++++ .../kleinanzeigen/search/SearchService.java | 35 +++++++++++++ .../telegram/TelegramService.java | 3 +- 10 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/Search.java create mode 100644 src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchCreate.java create mode 100644 src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchDto.java create mode 100644 src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchRepository.java create mode 100644 src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchService.java diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java index 7843dd6..db0ccac 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/KleinanzeigenApi.java @@ -1,7 +1,10 @@ package de.ph87.kleinanzeigen.kleinanzeigen; +import de.ph87.kleinanzeigen.kleinanzeigen.offer.LocationNotFound; import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferCreate; import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferService; +import de.ph87.kleinanzeigen.kleinanzeigen.search.SearchDto; +import de.ph87.kleinanzeigen.kleinanzeigen.search.SearchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; @@ -13,6 +16,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; import java.net.URI; +import java.time.DateTimeException; import java.util.concurrent.TimeUnit; @Slf4j @@ -21,25 +25,44 @@ import java.util.concurrent.TimeUnit; @RequiredArgsConstructor public class KleinanzeigenApi { - private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r%d"; + private static final String VERSCHENKEN_EPPELBORN_RADIUS = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/c192l339r%d"; + + private static final String QUERY_EPPELBORN_RADIUS = "https://www.kleinanzeigen.de/s-66571/preis:%s:%s/%s/k0l339r%d"; private final OfferService offerService; + private final SearchService searchService; + private final KleinanzeigenConfig config; @Scheduled(initialDelay = 0, fixedRate = 1, timeUnit = TimeUnit.MINUTES) public void fetch() { - final URI uri = URI.create(VERSCHENKEN_EPPELBORN_30KM.formatted(1, config.getRadiusKm())); + fetch(VERSCHENKEN_EPPELBORN_RADIUS.formatted(config.getRadiusKm()), config.getRadiusKm()); + searchService.findAllEnabledDto().forEach(this::fetch); + } + + private void fetch(final SearchDto search) { + final String url = QUERY_EPPELBORN_RADIUS.formatted( + search.getPriceMinEscaped(), + search.getPriceMaxEscaped(), + search.getQueryEscaped(), + search.getRadius() + ); + fetch(url, search.getRadius()); + } + + public void fetch(final String url, final int radius) { + final URI uri = URI.create(url); try { log.debug("Fetching page: {}", uri); final Document document = Jsoup.parse(uri.toURL(), 3000); - document.select("li.ad-listitem:not(.is-topad) article.aditem").forEach(article -> tryParse(article, uri)); + document.select("li.ad-listitem:not(.is-topad) article.aditem").forEach(article -> tryParse(article, uri, radius)); } catch (IOException e) { log.error("Failed to fetch Kleinanzeigen: {}", e.toString()); } } - private void tryParse(final Element article, final URI uri) { + private void tryParse(final Element article, final URI uri, final int radius) { try { final OfferCreate create = new OfferCreate(article, uri); if (create.getDistance() <= radius) { diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java index fc51727..b6adf32 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/Offer.java @@ -8,6 +8,7 @@ import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.ToString; +import java.math.BigDecimal; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; @@ -68,6 +69,13 @@ public class Offer { @Nullable private String imageURL = null; + @Column + @Nullable + private BigDecimal price = null; + + @Column(nullable = false) + private boolean negotiable; + @ToString.Exclude @OneToMany(mappedBy = "offer") private List messages = new ArrayList<>(); @@ -108,6 +116,14 @@ public class Offer { this.imageURL = create.getImageURL(); changed = true; } + if ((this.price == null) != (create.getPrice() == null) || (this.price != null && this.price.compareTo(create.getPrice()) != 0)) { + this.price = create.getPrice(); + changed = true; + } + if (!Objects.equals(this.negotiable, create.isNegotiable())) { + this.negotiable = create.isNegotiable(); + changed = true; + } return changed; } diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java index b1b5847..c663058 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferCreate.java @@ -10,6 +10,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import java.io.IOException; +import java.math.BigDecimal; import java.net.URI; import java.time.LocalDate; import java.time.LocalTime; @@ -52,7 +53,12 @@ public class OfferCreate { @Nullable private final String imageURL; - public OfferCreate(final Element article, final URI uri) { + @Nullable + private final BigDecimal price; + + private final boolean negotiable; + + public OfferCreate(final Element article, final URI uri) throws LocationNotFound { articleId = article.attr("data-adid"); title = article.select(".text-module-begin").text(); description = article.select(".aditem-main--middle--description").text(); @@ -67,6 +73,15 @@ public class OfferCreate { location = locationMatcher.group("location"); distance = Integer.parseInt(locationMatcher.group("distance")); imageURL = getImageURL(articleURL); + + final Matcher priceMatcher = Pattern.compile("(?\\d+(?:[,.]\\d+)?) €(? VB)?").matcher(article.select(".aditem-main--middle--price-shipping--price").text()); + if (priceMatcher.find()) { + price = new BigDecimal(priceMatcher.group("price")); + negotiable = priceMatcher.group("negotiable") != null; + } else { + price = null; + negotiable = false; + } } private ZonedDateTime parseDate(final String text) { diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java index f2b456d..ba26cf3 100644 --- a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/offer/OfferDto.java @@ -5,6 +5,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.ToString; +import java.math.BigDecimal; import java.time.ZonedDateTime; @Getter @@ -51,6 +52,11 @@ public class OfferDto { @Nullable private final String imageURL; + @Nullable + private final BigDecimal price; + + private final boolean negotiable; + public OfferDto(final @NonNull Offer offer) { id = offer.getId(); version = offer.getVersion(); @@ -66,8 +72,15 @@ public class OfferDto { description = offer.getDescription(); articleURL = offer.getArticleURL(); imageURL = offer.getImageURL(); + price = offer.getPrice(); + negotiable = offer.isNegotiable(); } + public String combinePrice() { + if (price == null) { + return "Zu Verschenken"; + } + return "%s €%s".formatted(price.intValue(), negotiable ? " VB" : ""); } } diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/Search.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/Search.java new file mode 100644 index 0000000..6e48a22 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/Search.java @@ -0,0 +1,49 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.search; + +import jakarta.annotation.Nullable; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; + +@Entity +@Getter +@ToString +@NoArgsConstructor +public class Search { + + @Id + @GeneratedValue + private long id; + + @Column(nullable = false) + private boolean enabled; + + @NonNull + @Column(nullable = false) + private String query; + + @Column(nullable = false) + private int radius; + + @Column + @Nullable + private Integer priceMin = null; + + @Column + @Nullable + private Integer priceMax = null; + + public Search(final SearchCreate create) { + enabled = create.isEnabled(); + query = create.getQuery(); + radius = create.getRadius(); + priceMin = create.getPriceMin(); + priceMax = create.getPriceMax(); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchCreate.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchCreate.java new file mode 100644 index 0000000..78c5121 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchCreate.java @@ -0,0 +1,31 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.search; + +import jakarta.annotation.Nullable; +import lombok.Data; +import lombok.NonNull; + +@Data +public class SearchCreate { + + private final boolean enabled; + + @NonNull + private final String query; + + private final int radius; + + @Nullable + private final Integer priceMin; + + @Nullable + private final Integer priceMax; + + public SearchCreate(final boolean enabled, @NonNull final String query, final int radius, @Nullable final Integer priceMin, @Nullable final Integer priceMax) { + this.enabled = enabled; + this.query = query; + this.radius = radius; + this.priceMin = priceMin; + this.priceMax = priceMax; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchDto.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchDto.java new file mode 100644 index 0000000..52a1ab8 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchDto.java @@ -0,0 +1,48 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.search; + +import jakarta.annotation.Nullable; +import lombok.Data; +import lombok.NonNull; + +import java.util.Locale; + +@Data +public class SearchDto { + + private final long id; + + private final boolean enabled; + + @NonNull + private final String query; + + private final int radius; + + @Nullable + private final Integer priceMin; + + @Nullable + private final Integer priceMax; + + public SearchDto(final Search search) { + this.id = search.getId(); + this.enabled = search.isEnabled(); + this.query = search.getQuery(); + this.radius = search.getRadius(); + this.priceMin = search.getPriceMin(); + this.priceMax = search.getPriceMax(); + } + + public String getQueryEscaped() { + return query.toLowerCase(Locale.ROOT).replaceAll("^[^a-z0-9]+|[^a-z0-9]+$", "").replaceAll("[^a-z0-9]+", "-"); + } + + public String getPriceMinEscaped() { + return priceMin == null ? "" : priceMin + ""; + } + + public String getPriceMaxEscaped() { + return priceMax == null ? "" : priceMax + ""; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchRepository.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchRepository.java new file mode 100644 index 0000000..05117d0 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchRepository.java @@ -0,0 +1,11 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.search; + +import org.springframework.data.repository.ListCrudRepository; + +import java.util.List; + +public interface SearchRepository extends ListCrudRepository { + + List findAllByEnabledTrue(); + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchService.java b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchService.java new file mode 100644 index 0000000..9dd97e7 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/kleinanzeigen/search/SearchService.java @@ -0,0 +1,35 @@ +package de.ph87.kleinanzeigen.kleinanzeigen.search; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Slf4j +@Service +@Transactional +@RequiredArgsConstructor +public class SearchService { + + private final SearchRepository searchRepository; + + @PostConstruct + public void init() { + if (searchRepository.count() == 0) { + searchRepository.save(new Search(new SearchCreate(true, "Garten Bank", 15, 0, 30))); + searchRepository.save(new Search(new SearchCreate(true, "Teich Pumpe", 15, 0, 30))); + } + } + + public List findAllEnabledDto() { + return searchRepository.findAllByEnabledTrue().stream().map(this::toDto).toList(); + } + + private SearchDto toDto(Search search) { + return new SearchDto(search); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java index 4a162d1..36ebc23 100644 --- a/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java +++ b/src/main/java/de/ph87/kleinanzeigen/telegram/TelegramService.java @@ -186,8 +186,9 @@ public class TelegramService { } private String createText(final OfferDto offer) { - return "%s\n%s\n%s\n%s\n".formatted( + return "%s\n%s\n%s %s (%d km)\n%s\n%s\n".formatted( offer.getTitle().replaceAll("\\[", "(").replaceAll("]", ")"), + offer.combinePrice(), offer.getZipcode(), offer.getLocation(), offer.getDistance(),