diff options
Diffstat (limited to 'rdns_scan')
195 files changed, 19145 insertions, 0 deletions
diff --git a/rdns_scan/zmap4rdns/.clang-format b/rdns_scan/zmap4rdns/.clang-format new file mode 100644 index 0000000..2c50773 --- /dev/null +++ b/rdns_scan/zmap4rdns/.clang-format @@ -0,0 +1,11 @@ +BasedOnStyle: LLVM +IndentWidth: 8 +UseTab: Always +BreakBeforeBraces: Linux +AllowShortIfStatementsOnASingleLine: false +IndentCaseLabels: false +DerivePointerAlignment: false +PointerAlignment: Right +BreakStringLiterals: false +SortIncludes: false +ReflowComments: false diff --git a/rdns_scan/zmap4rdns/.editorconfig b/rdns_scan/zmap4rdns/.editorconfig new file mode 100644 index 0000000..7c468e9 --- /dev/null +++ b/rdns_scan/zmap4rdns/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{c,h}] +indent_style = tab +indent_size = 8 + +[CMakeLists.txt] +indent_style = spaces +indent_size = 4 + +[*.py] +indent_style = spaces +indent_size = 4 + diff --git a/rdns_scan/zmap4rdns/CMakeLists.txt b/rdns_scan/zmap4rdns/CMakeLists.txt new file mode 100644 index 0000000..e2f2b75 --- /dev/null +++ b/rdns_scan/zmap4rdns/CMakeLists.txt @@ -0,0 +1,162 @@ +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +project(ZMAP C) +set(ZMAP_VERSION DEVELOPMENT) # Change DEVELOPMENT to version number for release + +option(ENABLE_DEVELOPMENT "Enable development specific compiler and linker flags" OFF) +option(ENABLE_LOG_TRACE "Enable log trace messages" OFF) +option(RESPECT_INSTALL_PREFIX_CONFIG "Respect CMAKE_INSTALL_PREFIX for /etc" OFF) +option(WITH_WERROR "Build with -Werror" OFF) +option(WITH_PFRING "Build with PF_RING ZC for send (10 GigE)" OFF) +option(FORCE_CONF_INSTALL "Overwrites existing configuration files at install" OFF) + +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(USING_CLANG "YES") +else() + set(USING_GCC "YES") +endif() + +if("${CMAKE_SYSTEM_NAME}" MATCHES "FreeBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "DragonFly") + set(BSD "YES") +endif() + +if("${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD") + set(NetBSD "YES") +endif() + +# Hardening and warnings for building with gcc +# Maybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations +set(GCCWARNINGS + "-Wall -Wformat=2 -Wno-format-nonliteral" + "-pedantic -fno-strict-aliasing" + "-Wextra" + "-Wfloat-equal -Wundef -Wwrite-strings -Wredundant-decls" + "-Wnested-externs -Wbad-function-cast -Winit-self" + "-Wmissing-noreturn" + "-Wstack-protector" +) + +# Fix line breaks +string(REPLACE ";" " " GCCWARNINGS "${GCCWARNINGS}") + +if(WITH_WERROR) + set(GCCWARNINGS "${GCCWARNINGS} -Werror") +endif() + +if(ENABLE_DEVELOPMENT) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ggdb") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g") +else() + # Hardening and optimizations for building with gcc + set(GCCHARDENING "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1") + if(NOT APPLE AND NOT BSD) + set(LDHARDENING "-z relro -z now") + else() + set(LDHARDENING "") + endif() + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCCHARDENING} -O2") + set(CMAKE_EXE_LINKER_FLAGS "${LDHARDENING} ${CMAKE_EXE_LINKER_FLAGS}") +endif() + +if(ENABLE_LOG_TRACE) + add_definitions("-DDEBUG") +endif() + + +set(CMAKE_C_FLAGS "${GCCWARNINGS} ${CMAKE_C_FLAGS}") + + +include(FindPkgConfig) +pkg_check_modules(JSON json-c) +if(JSON_FOUND) + include_directories(${JSON_INCLUDE_DIRS}) +else() + message(FATAL_ERROR "Did not find libjson") +endif() +string(REPLACE ";" " " JSON_CFLAGS "${JSON_CFLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${JSON_CFLAGS}") + +if(WITH_PFRING) + add_definitions("-DPFRING") + set(PFRING_LIBRARIES pfring rt numa) +endif() + +# Standard FLAGS +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") +if(NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") +endif() + +# Set up OS-specific include directories +if(APPLE) + if(EXISTS /opt/local/include) + include_directories(/opt/local/include) + endif() + if(EXISTS /opt/local/lib) + link_directories(/opt/local/lib) + endif() + if(EXISTS /usr/local/include) + include_directories(/usr/local/include) + endif() + if(EXISTS /usr/local/lib) + link_directories(/usr/local/lib) + endif() +endif() + +if(BSD) + include_directories(/usr/local/include) + link_directories(/usr/local/lib) +endif() + +if(NetBSD) + include_directories(/usr/pkg/include) + link_directories(/usr/pkg/lib) +endif() + +add_subdirectory(lib) +add_subdirectory(src) + +# Install conf files +if(RESPECT_INSTALL_PREFIX_CONFIG) + set(CONFIG_DESTINATION "etc/zmap") +else() + set(CONFIG_DESTINATION "/etc/zmap") +endif() + +FILE(GLOB CONF_FILES "${PROJECT_SOURCE_DIR}/conf/*") + +message(STATUS "Default ZMap configuration file location is /etc/zmap") +foreach(EACH_CONF ${CONF_FILES}) + get_filename_component(CONF_BASENAME ${EACH_CONF} NAME) + message(STATUS "Checking if ${CONF_BASENAME} exists there...") + if(NOT EXISTS "/etc/zmap/${CONF_BASENAME}") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + elseif(FORCE_CONF_INSTALL) + message(WARNING "FORCE_CONF_INSTALL will overwrite any existing configuration files") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + else() + message(WARNING "Existing configuration file detected at /etc/zmap/${CONF_BASENAME}, ${CONF_BASENAME} from sources will NOT be installed. Please check and install manually!") + endif() +endforeach() + +# Allow Debian Packaging +include(InstallRequiredSystemLibraries) + +set(CPACK_SET_DESTDIR "on") +set(CPACK_PACKAGING_INSTALL_PREFIX "/tmp") +set(CPACK_GENERATOR "DEB") + +set(CPACK_DEBIAN_PACKAGE_VERSION ${ZMAP_VERSION}) +set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional") +set(CPACK_DEBIAN_PACKAGE_SECTION "network") +set(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgmp10, libpcap0.8, libjson-c-dev") + +set(CPACK_PACKAGE_DESCRIPTION "Internet-scale network scanner") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "ZMap is an open source network scanner that enables researchers to easily perform Internet-wide network studies. With a single machine and a well provisioned network uplink, ZMap is capable of performing a complete scan of the IPv4 address space in under five minutes, approaching the theoretical limit of gigabit Ethernet. ZMap can be used to study protocol adoption over time, monitor service availability, and help us better understand large systems distributed across the Internet.") +set(CPACK_PACKAGE_CONTACT "Zakir Durumeric <[email protected]>") +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${VERSION}_${CPACK_DEBIAN_ARCHITECTURE}") + +set(CPACK_COMPONENTS_ALL Libraries ApplicationData) + +include(CPack) diff --git a/rdns_scan/zmap4rdns/INSTALL.md b/rdns_scan/zmap4rdns/INSTALL.md new file mode 100644 index 0000000..ecab3da --- /dev/null +++ b/rdns_scan/zmap4rdns/INSTALL.md @@ -0,0 +1,92 @@ +# Installing and Building ZMap + +## Installing via Package Manager + +ZMap operates on GNU/Linux, macOS, and BSD. The latest stable version (v2.1.1) +can be installed using most OS package managers: + +| OS | | +| ----------------------------------------- | ----------------------- | +| Fedora 19+ or EPEL 6+ | `sudo yum install zmap` | +| Debian 8+ or Ubuntu 14.04+ | `sudo apt install zmap` | +| Gentoo | `sudo emerge zmap` | +| macOS (using [Homebrew](https://brew.sh)) | `brew install zmap` | +| Arch Linux | `sudo pacman -S zmap` | + +## Building from Source + +### Installing ZMap Dependencies + +ZMap has the following dependencies: + + - [CMake](http://www.cmake.org/) - Cross-platform, open-source build system + - [GMP](http://gmplib.org/) - Free library for arbitrary precision arithmetic + - [gengetopt](http://www.gnu.org/software/gengetopt/gengetopt.html) - Command line option parsing for C programs + - [libpcap](http://www.tcpdump.org/) - Famous user-level packet capture library + - [flex](http://flex.sourceforge.net/) and [byacc](http://invisible-island.net/byacc/) - Output filter lexer and parser generator + - [json-c](https://github.com/json-c/json-c/) - JSON implementation in C + - [libunistring](https://www.gnu.org/software/libunistring/) - Unicode string library for C + - [libdnet](https://github.com/dugsong/libdnet) - (macOS Only) Gateway and route detection + +Install the required dependencies with the following commands. + +* On Debian-based systems (including Ubuntu): + ```sh + sudo apt-get install build-essential cmake libgmp3-dev gengetopt libpcap-dev flex byacc libjson-c-dev pkg-config libunistring-dev libmysqlclient-dev + ``` + +* On RHEL- and Fedora-based systems (including CentOS): + ```sh + sudo yum install cmake gmp-devel gengetopt libpcap-devel flex byacc json-c-devel libunistring-devel + ``` + +* On macOS systems (using [Homebrew](http://brew.sh/)): + ```sh + brew install pkg-config cmake gmp gengetopt json-c byacc libdnet libunistring + ``` + +* To launch a shell inside a Docker container with the build dependencies + mounted at `/src`: + ```sh + docker run -it -v $(pwd):/src zmap/builder + ``` + +### Building and Installing ZMap + +Once these prerequisites are installed, ZMap can be compiled by running: + ```sh + cmake . + make -j4 + ``` + +and then installed via `sudo make install`. + +### Development Notes + +- Enabling development turns on debug symbols, and turns off optimizations. +Release builds should be built with `-DENABLE_DEVELOPMENT=OFF`. + +- Enabling `log_trace` can have a major performance impact and should not be used +except during early development. Release builds should be built with `-DENABLE_LOG_TRACE=OFF`. + +- Building packages for some systems like Fedora and RHEL requires a user-definable +directory (buildroot) to put files. The way to respect this prefix is to run cmake +with `-DRESPECT_INSTALL_PREFIX_CONFIG=ON`. + +- Manpages (and their HTML representations) are generated from the `.ronn` source +files in the repository, using the [ronn](https://github.com/rtomayko/ronn) tool. +This does not happen automatically as part of the build process; to regenerate the +man pages you'll need to run `make manpages`. This target assumes that `ronn` is +in your PATH. + +- Building with some versions of CMake may fail with `unable to find parser.h`. +If this happens, try updating CMake. If it still fails, don't clone ZMap into a +path that contains the string `.com`, and try again. + +- ZMap may be installed to an alternative directory, with the `CMAKE_INSTALL_PREFIX` +option. For example, to install it in `$HOME/opt` run + ```sh + cmake -DCMAKE_INSTALL_PREFIX=$HOME/opt . + make -j4 + make install + ``` diff --git a/rdns_scan/zmap4rdns/checkFormat.sh b/rdns_scan/zmap4rdns/checkFormat.sh new file mode 100644 index 0000000..566bb48 --- /dev/null +++ b/rdns_scan/zmap4rdns/checkFormat.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +CLANG_FORMAT=clang-format-6.0 + +files_to_lint=$(find ./src ./lib -type f -name '*.c' -or -name '*.h') + +fail=0 +for f in ${files_to_lint}; do + d="$(diff -u "$f" <($CLANG_FORMAT -style=file "$f") || true)" + if ! [ -z "$d" ]; then + printf "The file %s is not compliant with the coding style:\n%s\n" "$f" "$d" + fail=1 + fi +done + +if [ "$fail" -eq "1" ]; then + if [ ! -z $ZMAP_ENFORCE_FORMAT ]; then + exit 1 + fi +fi diff --git a/rdns_scan/zmap4rdns/conf/blocklist.conf b/rdns_scan/zmap4rdns/conf/blocklist.conf new file mode 100644 index 0000000..018b0f4 --- /dev/null +++ b/rdns_scan/zmap4rdns/conf/blocklist.conf @@ -0,0 +1,25 @@ +# From IANA IPv4 Special-Purpose Address Registry +# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +# Updated 2013-05-22 + +0.0.0.0/8 # RFC1122: "This host on this network" +10.0.0.0/8 # RFC1918: Private-Use +100.64.0.0/10 # RFC6598: Shared Address Space +127.0.0.0/8 # RFC1122: Loopback +169.254.0.0/16 # RFC3927: Link Local +172.16.0.0/12 # RFC1918: Private-Use +192.0.0.0/24 # RFC6890: IETF Protocol Assignments +192.0.2.0/24 # RFC5737: Documentation (TEST-NET-1) +192.88.99.0/24 # RFC3068: 6to4 Relay Anycast +192.168.0.0/16 # RFC1918: Private-Use +198.18.0.0/15 # RFC2544: Benchmarking +198.51.100.0/24 # RFC5737: Documentation (TEST-NET-2) +203.0.113.0/24 # RFC5737: Documentation (TEST-NET-3) +240.0.0.0/4 # RFC1112: Reserved +255.255.255.255/32 # RFC0919: Limited Broadcast + +# From IANA Multicast Address Space Registry +# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml +# Updated 2013-06-25 + +224.0.0.0/4 # RFC5771: Multicast/Reserved diff --git a/rdns_scan/zmap4rdns/conf/zmap.conf b/rdns_scan/zmap4rdns/conf/zmap.conf new file mode 100644 index 0000000..817c184 --- /dev/null +++ b/rdns_scan/zmap4rdns/conf/zmap.conf @@ -0,0 +1,22 @@ +### Probe Module to use +#probe-module tcp_synscan + + +### Destination port to scan +#target-port 443 + +### Scan rate in packets/sec +#rate 10000 + +### Scan rate in bandwidth (bits/sec); overrides `rate` +#bandwidth 1M # 1mbps + + +### Blocklist file to use. We encourage you to exclude +### RFC1918, IANA reserved, and multicast networks, +### in addition to those who have opted out of your +### network scans. +blocklist-file "/etc/zmap/blocklist.conf" + +### Optionally print a summary at the end +#summary diff --git a/rdns_scan/zmap4rdns/containers/build-push-builder.sh b/rdns_scan/zmap4rdns/containers/build-push-builder.sh new file mode 100644 index 0000000..ca388ea --- /dev/null +++ b/rdns_scan/zmap4rdns/containers/build-push-builder.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +ZMAP_CONTAINER_TAG=${ZMAP_CONTAINER_TAG:-'latest'} +docker build -f builder.dockerfile -t zmap/builder:$ZMAP_CONTAINER_TAG . +docker push zmap/builder:$ZMAP_CONTAINER_TAG diff --git a/rdns_scan/zmap4rdns/containers/builder.dockerfile b/rdns_scan/zmap4rdns/containers/builder.dockerfile new file mode 100644 index 0000000..7593704 --- /dev/null +++ b/rdns_scan/zmap4rdns/containers/builder.dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update -y --quiet +RUN apt-get install -y -qq \ + build-essential \ + byacc \ + cmake \ + flex \ + gengetopt \ + libgmp3-dev \ + libjson-c-dev \ + libpcap-dev \ + libunistring-dev \ + pkg-config \ + python3 diff --git a/rdns_scan/zmap4rdns/examples/forge-socket/README b/rdns_scan/zmap4rdns/examples/forge-socket/README new file mode 100644 index 0000000..05282ab --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/forge-socket/README @@ -0,0 +1,4 @@ +Forge Socket +============ + +Forge Socket is now maintained at https://github.com/ewust/forge_socket. diff --git a/rdns_scan/zmap4rdns/examples/probe-modules/module_tcp_cisco_backdoor.c b/rdns_scan/zmap4rdns/examples/probe-modules/module_tcp_cisco_backdoor.c new file mode 100644 index 0000000..9cd7bde --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/probe-modules/module_tcp_cisco_backdoor.c @@ -0,0 +1,203 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +probe_module_t module_tcp_synscan; +static uint32_t num_ports; + +static int synscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + __attribute__((unused)) void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct tcphdr)); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, dst_port, TH_SYN); + return EXIT_SUCCESS; +} + +// instead of settings sequence number to be random for validation +// let's instead set to something static so that we can easily +// set acknowledgement number. I don't know how integer overflow +// is going to act in this. +// uint32_t tcp_seq = validation[0]; +// From Mandiant +// 1. To initiate the process, a uniquely crafted TCP SYN packet is sent +// to port 80 of the “implanted” router. It is important to note that +// the difference between the sequence and acknowledgment numbers must +// be set to 0xC123D. Also the ACK number doesn’t need to be zero. + +#define BACKDOOR_SEQ 0x3D120C00 +//#define BACKDOOR_SEQ 0x000C123D // wrong byte order +#define BACKDOOR_ACK 0x0 +#define EXPECTED_RESPONSE_SEQ 0 +//#define EXPECTED_RESPONSE_ACK 0x000C123E // wrong byte order +#define EXPECTED_RESPONSE_ACK 0x3E120C00 + +static int synscan_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + tcp_header->th_sport = + htons(get_src_port(num_ports, probe_num, validation)); + tcp_header->th_seq = BACKDOOR_SEQ; + tcp_header->th_ack = BACKDOOR_ACK; + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp_checksum(sizeof(struct tcphdr), ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + return EXIT_SUCCESS; +} + +static void synscan_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct tcphdr *tcph = (struct tcphdr *)&iph[1]; + fprintf(fp, + "tcp { source: %u | dest: %u | seq: %u | checksum: %#04X }\n", + ntohs(tcph->th_sport), ntohs(tcph->th_dport), + ntohl(tcph->th_seq), ntohs(tcph->th_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + __attribute__((unused)) uint32_t *src_ip, + uint32_t *validation) +{ + if (ip_hdr->ip_p != IPPROTO_TCP) { + return 0; + } + if ((4 * ip_hdr->ip_hl + sizeof(struct tcphdr)) > len) { + // buffer not large enough to contain expected tcp header + return 0; + } + struct tcphdr *tcp = + (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t sport = tcp->th_sport; + uint16_t dport = tcp->th_dport; + // validate source port + if (ntohs(sport) != zconf.target_port) { + return 0; + } + // validate destination port + if (!check_dst_port(ntohs(dport), num_ports, validation)) { + return 0; + } + // DO NOT validate ack number as this is currently statically set + // validate tcp acknowledgement number + // if (htonl(tcp->th_ack) != htonl(validation[0])+1) { + // return 0; + //} + return 1; +} + +static void synscan_process_packet(const u_char *packet, uint32_t len, + fieldset_t *fs, + __attribute__((unused)) uint32_t *validation, + __attribute__((unused)) struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct tcphdr *tcp = + (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + fs_add_uint64(fs, "urgentptr", (uint64_t)ntohs(tcp->th_urp)); + fs_add_uint64(fs, "flags", (uint64_t)ntohs(tcp->th_flags)); + fs_add_binary(fs, "raw", len, (void *)packet, 0); + + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_string(fs, "classification", (char *)"rst", 0); + fs_add_bool(fs, "success", 0); + } else if (tcp->th_seq == EXPECTED_RESPONSE_SEQ && tcp->th_urp) { + fs_add_string(fs, "classification", (char *)"backdoor", 0); + fs_add_bool(fs, "success", 1); + } else { // SYNACK packet + fs_add_string(fs, "classification", (char *)"synack", 0); + fs_add_bool(fs, "success", 1); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + {.name = "urgentptr", .type = "int", .desc = "Urgent POinter"}, + {.name = "flags", .type = "int", .desc = "tcp flags"}, + {.name = "raw", .type = "binary", .desc = "raw packet"}, + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}}; + +probe_module_t module_tcp_cisco_backdoor = { + .name = "tcp_cisco_backdoor", + .packet_length = 54, + .pcap_filter = "tcp && tcp[13] & 4 != 0 || tcp[13] == 18", + .pcap_snaplen = 256, + .port_args = 1, + .global_initialize = &synscan_global_initialize, + .thread_initialize = &synscan_init_perthread, + .make_packet = &synscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synscan_process_packet, + .validate_packet = &synscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 10}; diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/README b/rdns_scan/zmap4rdns/examples/udp-probes/README new file mode 100644 index 0000000..b925b51 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/README @@ -0,0 +1,51 @@ + +UDP Data Probes +====== + +This directory contains a set of data files that can be used with the UDP probe module. + + +USING: +----- + +$ zmap -M udp -p 137 --probe-args=file:examples/udp-probes/netbios_137.pkt + + +PROBES: +----- + +citrix_1604.pkt This probe triggers a response from Citrix application discovery services on UDP port 1604 +db2disco_523.pkt This probe triggers a response from IBM DB2 discovery services on UDP port 523 +digi1_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (default magic) +digi2_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (devkit magic) +digi3_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (oem magic) +dns_53.pkt This probe queries for the DNS vendor and version using the BIND version TXT record over UDP port 53 +dns_53_queryAwww.google.it.pkt This probe queries for the domain www.google.it A record over UDP port 53 +dns_53_queryAwww.google.com.pkt This probe queries for the domain www.google.com A record over UDP port 53 +ipmi_623.pkt This probe triggers a Get Channel Authentication reply from IPMI endpoints on UDP port 623 +mdns_5353.pkt This probe triggers a response from mDNS/Avahi/Bonjour discovery services on UDP port 5353 +memcache_11211.pkt This probe triggers a response from memcached on UDP port 11211 (stats items). +mssql_1434.pkt This probe triggers a response from Microsoft SQL Server discovery services on UDP port 1434 +natpmp_5351.pkt This probe triggers a response from NATPMP-enabled devices on UDP port 5351 +netbios_137.pkt This probe triggers a status reply from NetBIOS services on UDP port 137 +ntp_123.pkt This probe triggers a response from NTP services on UDP port 123 +ntp_123_monlist.pkt This probe triggers a response for command "monlist" from NTP services on UDP port 123 +pca_nq_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (network query) +pca_st_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (status) +portmap_111.pkt This probe triggers a response from SunRPC portmapper services on UDP port 111 +ripv1_520.pkt This probe triggers a response from the RIPv1 enabled routers/devices on UDP port 520 +sentinel_5093.pkt This probe triggers a response from the Sentinel license manager service on UDP port 5093 +snmp1_161.pkt This probe queries for the system description field of SNMP v1 services using community string public over UDP port 161 +snmp2_161.pkt This probe queries for the system description field of SNMP v2 services using community string public over UDP port 161 +snmp3_161.pkt This probe triggers a response from SNMP v3 services on UDP port 161 +upnp_1900.pkt This probe triggers a response from UPnP SSDP services on UDP port 1900 +wdbrpc_17185.pkt This probe triggers a response from VxWorks WDBRPC services on UDP port 17185 +wsd_3702.pkt This probe triggers a response from WSD/DPWS services on UDP port 3702 +coap_5683.pkt This probe triggers a response from COAP services on UDP port 5683 + +NOTES: +----- + +Most of these probes return useful data in the response. Parsing this data requires capturing the raw output +and decoding this using a protocol-specific dissector. In most cases, Wireshark is capable of decoding these +replies. diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ard_3283.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ard_3283.pkt Binary files differnew file mode 100644 index 0000000..89f6baf --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ard_3283.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_47808.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_47808.pkt Binary files differnew file mode 100644 index 0000000..6986139 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_47808.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_rpm_47808.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_rpm_47808.pkt Binary files differnew file mode 100644 index 0000000..041c956 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/bacnet_rpm_47808.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/chargen_19.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/chargen_19.pkt new file mode 100644 index 0000000..6b2aaa7 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/chargen_19.pkt @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/citrix_1604.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/citrix_1604.pkt Binary files differnew file mode 100644 index 0000000..acc5a73 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/citrix_1604.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/coap_5683.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/coap_5683.pkt new file mode 100644 index 0000000..da3b55e --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/coap_5683.pkt @@ -0,0 +1 @@ +@}p�.well-knowncore
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/db2_523.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/db2_523.pkt Binary files differnew file mode 100644 index 0000000..bdea77a --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/db2_523.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/db2disco_523.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/db2disco_523.pkt Binary files differnew file mode 100644 index 0000000..3883745 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/db2disco_523.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/digi1_2362.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/digi1_2362.pkt Binary files differnew file mode 100644 index 0000000..74f57b8 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/digi1_2362.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/digi2_2362.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/digi2_2362.pkt Binary files differnew file mode 100644 index 0000000..d962606 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/digi2_2362.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/digi3_2362.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/digi3_2362.pkt Binary files differnew file mode 100644 index 0000000..ffba125 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/digi3_2362.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/dns_53.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53.pkt Binary files differnew file mode 100644 index 0000000..616e17e --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.com.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.com.pkt Binary files differnew file mode 100644 index 0000000..9392e73 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.com.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.it.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.it.pkt Binary files differnew file mode 100644 index 0000000..99238c0 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/dns_53_queryAwww.google.it.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ipmi_623.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ipmi_623.pkt Binary files differnew file mode 100644 index 0000000..025d0fa --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ipmi_623.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ldap_389.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ldap_389.pkt Binary files differnew file mode 100644 index 0000000..06600df --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ldap_389.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/mdns_5353.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/mdns_5353.pkt Binary files differnew file mode 100644 index 0000000..8221eae --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/mdns_5353.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/memcache_11211.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/memcache_11211.pkt Binary files differnew file mode 100644 index 0000000..0c8d99c --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/memcache_11211.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/mssql_1434.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/mssql_1434.pkt new file mode 100644 index 0000000..25cb955 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/mssql_1434.pkt @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/nat_port_mapping_5351.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/nat_port_mapping_5351.pkt Binary files differnew file mode 100644 index 0000000..3892404 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/nat_port_mapping_5351.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/natpmp_5351.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/natpmp_5351.pkt Binary files differnew file mode 100644 index 0000000..593f470 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/natpmp_5351.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/netbios_137.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/netbios_137.pkt Binary files differnew file mode 100644 index 0000000..5177cf6 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/netbios_137.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/netis_53413.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/netis_53413.pkt Binary files differnew file mode 100644 index 0000000..3edbb54 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/netis_53413.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123.pkt Binary files differnew file mode 100644 index 0000000..ef66e7e --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123_monlist.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123_monlist.pkt Binary files differnew file mode 100644 index 0000000..ba2912d --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ntp_123_monlist.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/openvpn_1194.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/openvpn_1194.pkt Binary files differnew file mode 100644 index 0000000..fec8aa2 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/openvpn_1194.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/pca_nq_5632.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/pca_nq_5632.pkt new file mode 100644 index 0000000..8d51173 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/pca_nq_5632.pkt @@ -0,0 +1 @@ +NQ
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/pca_st_5632.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/pca_st_5632.pkt new file mode 100644 index 0000000..86aa1fa --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/pca_st_5632.pkt @@ -0,0 +1 @@ +ST
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/pcanywhere_5632.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/pcanywhere_5632.pkt Binary files differnew file mode 100644 index 0000000..a5c6833 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/pcanywhere_5632.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/portmap_111.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/portmap_111.pkt Binary files differnew file mode 100644 index 0000000..7cfbdb9 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/portmap_111.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/qotd_17.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/qotd_17.pkt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/qotd_17.pkt @@ -0,0 +1 @@ + diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/rdp_3389.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/rdp_3389.pkt Binary files differnew file mode 100644 index 0000000..2423120 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/rdp_3389.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ripv1_520.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ripv1_520.pkt Binary files differnew file mode 100644 index 0000000..56ea97b --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ripv1_520.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/sentinel_5093.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/sentinel_5093.pkt Binary files differnew file mode 100644 index 0000000..158b3a9 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/sentinel_5093.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/sip_5060.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/sip_5060.pkt new file mode 100644 index 0000000..62f9bda --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/sip_5060.pkt @@ -0,0 +1 @@ +OPTIONS
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/sip_options.tpl b/rdns_scan/zmap4rdns/examples/udp-probes/sip_options.tpl new file mode 100644 index 0000000..c1cdafc --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/sip_options.tpl @@ -0,0 +1,12 @@ +OPTIONS sip:${RAND_ALPHA=8}@${DADDR} SIP/2.0 +Via: SIP/2.0/UDP ${SADDR}:${SPORT};branch=${RAND_ALPHA=6}.${RAND_DIGIT=10};rport;alias +From: sip:${RAND_ALPHA=8}@${SADDR}:${SPORT};tag=${RAND_DIGIT=8} +To: sip:${RAND_ALPHA=8}@${DADDR} +Call-ID: ${RAND_DIGIT=10}@${SADDR} +CSeq: 1 OPTIONS +Contact: sip:${RAND_ALPHA=8}@${SADDR}:${SPORT} +Content-Length: 0 +Max-Forwards: 20 +User-Agent: ${RAND_ALPHA=8} +Accept: text/plain + diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/snmp1_161.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/snmp1_161.pkt Binary files differnew file mode 100644 index 0000000..209e3cf --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/snmp1_161.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/snmp2_161.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/snmp2_161.pkt Binary files differnew file mode 100644 index 0000000..6c10b43 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/snmp2_161.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/snmp3_161.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/snmp3_161.pkt Binary files differnew file mode 100644 index 0000000..ab735cb --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/snmp3_161.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ssdp_1900.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ssdp_1900.pkt new file mode 100644 index 0000000..8ec2c66 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ssdp_1900.pkt @@ -0,0 +1,5 @@ +M-SEARCH * HTTP/1.1 +HOST:239.255.255.250:1900 +ST:ssdp:all +MAN:"ssdp:discover" + diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/tftp_69.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/tftp_69.pkt Binary files differnew file mode 100644 index 0000000..26ee37f --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/tftp_69.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_10001.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_10001.pkt Binary files differnew file mode 100644 index 0000000..2420198 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_10001.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v1_10001.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v1_10001.pkt Binary files differnew file mode 100644 index 0000000..f66c9cf --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v1_10001.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v2_10001.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v2_10001.pkt Binary files differnew file mode 100644 index 0000000..b6abc99 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/ubiquiti_discovery_v2_10001.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/upnp_1900.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/upnp_1900.pkt new file mode 100644 index 0000000..f56f7f5 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/upnp_1900.pkt @@ -0,0 +1,7 @@ +M-SEARCH * HTTP/1.1 +Host:239.255.255.250:1900 +ST:upnp:rootdevice +Man:"ssdp:discover" +MX:3 + + diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/valve_27015.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/valve_27015.pkt Binary files differnew file mode 100644 index 0000000..0f61da9 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/valve_27015.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/wdbrpc_17185.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/wdbrpc_17185.pkt Binary files differnew file mode 100644 index 0000000..2d0a676 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/wdbrpc_17185.pkt diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/wsd_3702.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/wsd_3702.pkt new file mode 100644 index 0000000..87704a5 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/wsd_3702.pkt @@ -0,0 +1,3 @@ +<?xml version="1.0" encoding="utf-8"?> +<soap:Envelope xmlns:soap="http://www.w3.org/2003/05/soap-envelope" xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing" xmlns:wsd="http://schemas.xmlsoap.org/ws/2005/04/discovery" xmlns:wsdp="http://schemas.xmlsoap.org/ws/2006/02/devprof"> +<soap:Header><wsa:To>urn:schemas-xmlsoap-org:ws:2005:04:discovery</wsa:To><wsa:Action>http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe</wsa:Action><wsa:MessageID>urn:uuid:ce04dad0-5d2c-4026-9146-1aabfc1e4111</wsa:MessageID></soap:Header><soap:Body><wsd:Probe><wsd:Types>wsdp:Device</wsd:Types></wsd:Probe></soap:Body></soap:Envelope> diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/wsd_malformed_3702.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/wsd_malformed_3702.pkt new file mode 100644 index 0000000..9fd51b3 --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/wsd_malformed_3702.pkt @@ -0,0 +1 @@ +<:/> diff --git a/rdns_scan/zmap4rdns/examples/udp-probes/xdmcp_177.pkt b/rdns_scan/zmap4rdns/examples/udp-probes/xdmcp_177.pkt Binary files differnew file mode 100644 index 0000000..96a017a --- /dev/null +++ b/rdns_scan/zmap4rdns/examples/udp-probes/xdmcp_177.pkt diff --git a/rdns_scan/zmap4rdns/format.sh b/rdns_scan/zmap4rdns/format.sh new file mode 100644 index 0000000..7380192 --- /dev/null +++ b/rdns_scan/zmap4rdns/format.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +set -o pipefail + +MAJOR_REV=$((clang-format --version | awk '{print $3}' | cut -d '.' -f 1) || echo 0) +if [ $MAJOR_REV -lt 5 ]; then + echo "error: need at least clang-format version 5.x" + exit 1 +fi + +FORMAT_CMD="clang-format -i -style=file" + +# No files passed, format everything +if [ $# -eq 0 ]; then + echo "formatting all C code in src/ and lib/" + find ./src -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./src -type f -name '*.h' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.h' -exec $FORMAT_CMD {} \; + exit 0 +fi + +# File names passed, format only those files +$FORMAT_CMD $@ diff --git a/rdns_scan/zmap4rdns/lib/CMakeLists.txt b/rdns_scan/zmap4rdns/lib/CMakeLists.txt new file mode 100644 index 0000000..e92e9b8 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/CMakeLists.txt @@ -0,0 +1,17 @@ +SET(LIB_SOURCES + blocklist.c + constraint.c + logger.c + pbm.c + random.c + rijndael-alg-fst.c + xalloc.c + lockfd.c + util.c + queue.c + csv.c +) + +add_library(zmaplib STATIC ${LIB_SOURCES}) + +target_include_directories (zmaplib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/rdns_scan/zmap4rdns/lib/blocklist.c b/rdns_scan/zmap4rdns/lib/blocklist.c new file mode 100644 index 0000000..c12425a --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/blocklist.c @@ -0,0 +1,285 @@ +/* + * Blocklist Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "blocklist.h" + +#include <errno.h> +#include <stdio.h> +#include <assert.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> + +#include "constraint.h" +#include "logger.h" +#include "xalloc.h" + +#define ADDR_DISALLOWED 0 +#define ADDR_ALLOWED 1 + +typedef struct bl_linked_list { + bl_cidr_node_t *first; + bl_cidr_node_t *last; + uint32_t len; +} bl_ll_t; + +static constraint_t *constraint = NULL; + +// keep track of the prefixes we've tried to BL/WL +// for logging purposes +static bl_ll_t *blocklisted_cidrs = NULL; +static bl_ll_t *allowlisted_cidrs = NULL; + +void bl_ll_add(bl_ll_t *l, struct in_addr addr, uint16_t p) +{ + assert(l); + bl_cidr_node_t *new = xmalloc(sizeof(bl_cidr_node_t)); + new->next = NULL; + new->ip_address = addr.s_addr; + new->prefix_len = p; + if (!l->first) { + l->first = new; + } else { + l->last->next = new; + } + l->last = new; + l->len++; +} + +bl_cidr_node_t *get_blocklisted_cidrs(void) { return blocklisted_cidrs->first; } + +bl_cidr_node_t *get_allowlisted_cidrs(void) { return allowlisted_cidrs->first; } + +uint32_t blocklist_lookup_index(uint64_t index) +{ + return ntohl(constraint_lookup_index(constraint, index, ADDR_ALLOWED)); +} + +// check whether a single IP address is allowed to be scanned. +// 1 => is allowed +// 0 => is not allowed +int blocklist_is_allowed(uint32_t s_addr) +{ + return constraint_lookup_ip(constraint, ntohl(s_addr)) == ADDR_ALLOWED; +} + +static void _add_constraint(struct in_addr addr, int prefix_len, int value) +{ + constraint_set(constraint, ntohl(addr.s_addr), prefix_len, value); + if (value == ADDR_ALLOWED) { + bl_ll_add(allowlisted_cidrs, addr, prefix_len); + } else if (value == ADDR_DISALLOWED) { + bl_ll_add(blocklisted_cidrs, addr, prefix_len); + } else { + log_fatal("blocklist", + "unknown type of blocklist operation specified"); + } +} + +// blocklist a CIDR network allocation +// e.g. blocklist_add("128.255.134.0", 24) +void blocklist_prefix(char *ip, int prefix_len) +{ + struct in_addr addr; + addr.s_addr = inet_addr(ip); + _add_constraint(addr, prefix_len, ADDR_DISALLOWED); +} + +// allowlist a CIDR network allocation +void allowlist_prefix(char *ip, int prefix_len) +{ + struct in_addr addr; + addr.s_addr = inet_addr(ip); + _add_constraint(addr, prefix_len, ADDR_ALLOWED); +} + +static int init_from_string(char *ip, int value) +{ + int prefix_len = 32; + char *slash = strchr(ip, '/'); + if (slash) { // split apart network and prefix length + *slash = '\0'; + char *end; + char *len = slash + 1; + errno = 0; + prefix_len = strtol(len, &end, 10); + if (end == len || errno != 0 || prefix_len < 0 || + prefix_len > 32) { + log_fatal("constraint", + "'%s' is not a valid prefix length", len); + return -1; + } + } + struct in_addr addr; + int ret = -1; + if (inet_aton(ip, &addr) == 0) { + // Not an IP and not a CIDR block, try dns resolution + struct addrinfo hint, *res; + memset(&hint, 0, sizeof(hint)); + hint.ai_family = PF_INET; + int r = getaddrinfo(ip, NULL, &hint, &res); + if (r) { + log_error("constraint", + "'%s' is not a valid IP " + "address or hostname", + ip); + return -1; + } + // Got some addrinfo, let's see what happens + for (struct addrinfo *aip = res; aip; aip = aip->ai_next) { + if (aip->ai_family != AF_INET) { + continue; + } + struct sockaddr_in *sa = + (struct sockaddr_in *)aip->ai_addr; + memcpy(&addr, &sa->sin_addr, sizeof(addr)); + log_debug("constraint", "%s retrieved by hostname", + inet_ntoa(addr)); + ret = 0; + _add_constraint(addr, prefix_len, value); + } + } else { + _add_constraint(addr, prefix_len, value); + return 0; + } + return ret; +} + +static int init_from_file(char *file, const char *name, int value, + int ignore_invalid_hosts) +{ + FILE *fp; + char line[1000]; + + fp = fopen(file, "r"); + if (fp == NULL) { + log_fatal(name, "unable to open %s file: %s: %s", name, file, + strerror(errno)); + } + + while (fgets(line, sizeof(line), fp) != NULL) { + char *comment = strchr(line, '#'); + if (comment) { + *comment = '\0'; + } + // hostnames can be up to 255 bytes + char ip[256]; + if ((sscanf(line, "%256s", ip)) == EOF) { + continue; + } + if (init_from_string(ip, value)) { + if (!ignore_invalid_hosts) { + log_fatal(name, "unable to parse %s file: %s", + name, file); + } + } + } + fclose(fp); + + return 0; +} + +static void init_from_array(char **cidrs, size_t len, int value, + int ignore_invalid_hosts) +{ + for (int i = 0; i < (int)len; i++) { + int ret = init_from_string(cidrs[i], value); + if (ret && !ignore_invalid_hosts) { + log_fatal("constraint", + "Unable to init from CIDR list"); + } + } +} + +uint64_t blocklist_count_allowed(void) +{ + assert(constraint); + return constraint_count_ips(constraint, ADDR_ALLOWED); +} + +uint64_t blocklist_count_not_allowed(void) +{ + assert(constraint); + return constraint_count_ips(constraint, ADDR_DISALLOWED); +} + +// network order +uint32_t blocklist_ip_to_index(uint32_t ip) +{ + assert(constraint); + uint32_t ip_hostorder = ntohl(ip); + return constraint_lookup_ip(constraint, ip_hostorder); +} + +// Initialize address constraints from allowlist and blocklist files. +// Either can be set to NULL to omit. +int blocklist_init(char *allowlist_filename, char *blocklist_filename, + char **allowlist_entries, size_t allowlist_entries_len, + char **blocklist_entries, size_t blocklist_entries_len, + int ignore_invalid_hosts) +{ + assert(!constraint); + + blocklisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + allowlisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + + if (allowlist_filename && allowlist_entries) { + log_warn("allowlist", + "both a allowlist file and destination addresses " + "were specified. The union of these two sources " + "will be utilized."); + } + if (allowlist_filename || allowlist_entries_len > 0) { + // using a allowlist, so default to allowing nothing + constraint = constraint_init(ADDR_DISALLOWED); + log_debug("constraint", "blocklisting 0.0.0.0/0"); + if (allowlist_filename) { + init_from_file(allowlist_filename, "allowlist", + ADDR_ALLOWED, ignore_invalid_hosts); + } + if (allowlist_entries) { + init_from_array(allowlist_entries, + allowlist_entries_len, ADDR_ALLOWED, + ignore_invalid_hosts); + } + } else { + // no allowlist, so default to allowing everything + log_debug("blocklist", + "no allowlist file or allowlist entries provided"); + constraint = constraint_init(ADDR_ALLOWED); + } + if (blocklist_filename) { + init_from_file(blocklist_filename, "blocklist", ADDR_DISALLOWED, + ignore_invalid_hosts); + } + if (blocklist_entries) { + init_from_array(blocklist_entries, blocklist_entries_len, + ADDR_DISALLOWED, ignore_invalid_hosts); + } + init_from_string(strdup("0.0.0.0"), ADDR_DISALLOWED); + constraint_paint_value(constraint, ADDR_ALLOWED); + uint64_t allowed = blocklist_count_allowed(); + log_debug("constraint", + "%lu addresses (%0.0f%% of address " + "space) can be scanned", + allowed, allowed * 100. / ((long long int)1 << 32)); + if (!allowed) { + log_error("blocklist", + "no addresses are eligible to be scanned in the " + "current configuration. This may be because the " + "blocklist being used by ZMap (%s) prevents " + "any addresses from receiving probe packets.", + blocklist_filename); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/lib/blocklist.h b/rdns_scan/zmap4rdns/lib/blocklist.h new file mode 100644 index 0000000..5ae566c --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/blocklist.h @@ -0,0 +1,50 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <stdint.h> + +#ifndef BLACKLIST_H +#define BLACKLIST_H + +typedef struct bl_cidr_node { + uint32_t ip_address; + int prefix_len; + struct bl_cidr_node *next; +} bl_cidr_node_t; + +uint32_t blocklist_lookup_index(uint64_t index); + +int blocklist_is_allowed(uint32_t s_addr); + +void blocklist_prefix(char *ip, int prefix_len); + +void allowlist_prefix(char *ip, int prefix_len); + +int blocklist_init(char *allowlist, char *blocklist, char **allowlist_entries, + size_t allowlist_entries_len, char **blocklist_entries, + size_t blocklist_entries_len, int ignore_invalid_hosts); + +uint64_t blocklist_count_allowed(); + +uint64_t blocklist_count_not_allowed(); + +uint32_t blocklist_ip_to_index(uint32_t ip); + +bl_cidr_node_t *get_blocklisted_cidrs(void); +bl_cidr_node_t *get_allowlisted_cidrs(void); + +#endif diff --git a/rdns_scan/zmap4rdns/lib/constraint.c b/rdns_scan/zmap4rdns/lib/constraint.c new file mode 100644 index 0000000..2c1c749 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/constraint.c @@ -0,0 +1,416 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <errno.h> +#include <assert.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "../lib/constraint.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +// +// Efficient address-space constraints (AH 7/2013) +// +// This module uses a tree-based representation to efficiently +// manipulate and query constraints on the address space to be +// scanned. It provides a value for every IP address, and these +// values are applied by setting them for network prefixes. Order +// matters: setting a value replaces any existing value for that +// prefix or subsets of it. We use this to implement network +// allowlisting and blocklisting. +// +// Think of setting values in this structure like painting +// subnets with different colors. We can paint subnets black to +// exclude them and white to allow them. Only the top color shows. +// This makes for potentially very powerful constraint specifications. +// +// Internally, this is implemented using a binary tree, where each +// node corresponds to a network prefix. (E.g., the root is +// 0.0.0.0/0, and its children, if present, are 0.0.0.0/1 and +// 128.0.0.0/1.) Each leaf of the tree stores the value that applies +// to every address within the leaf's portion of the prefix space. +// +// As an optimization, after all values are set, we look up the +// value or subtree for every /16 prefix and cache them as an array. +// This lets subsequent lookups bypass the bottom half of the tree. +// + +/* + * Constraint Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +typedef struct node { + struct node *l; + struct node *r; + value_t value; + uint64_t count; +} node_t; + +// As an optimization, we precompute lookups for every prefix of this +// length: +#define RADIX_LENGTH 20 + +struct _constraint { + node_t *root; // root node of the tree + uint32_t *radix; // array of prefixes (/RADIX_LENGTH) that are painted + // paint_value + size_t radix_len; // number of prefixes in radix array + int painted; // have we precomputed counts for each node? + value_t paint_value; // value for which we precomputed counts +}; + +// Tree operations respect the invariant that every node that isn't a +// leaf has exactly two children. +#define IS_LEAF(node) ((node)->l == NULL) + +// Allocate a new leaf with the given value +static node_t *_create_leaf(value_t value) +{ + node_t *node = xmalloc(sizeof(node_t)); + node->l = NULL; + node->r = NULL; + node->value = value; + return node; +} + +// Free the subtree rooted at node. +static void _destroy_subtree(node_t *node) +{ + if (node == NULL) + return; + _destroy_subtree(node->l); + _destroy_subtree(node->r); + free(node); +} + +// Convert from an internal node to a leaf. +static void _convert_to_leaf(node_t *node) +{ + assert(node); + assert(!IS_LEAF(node)); + _destroy_subtree(node->l); + _destroy_subtree(node->r); + node->l = NULL; + node->r = NULL; +} + +// Recursive function to set value for a given network prefix within +// the tree. (Note: prefix must be in host byte order.) +static void _set_recurse(node_t *node, uint32_t prefix, int len, value_t value) +{ + assert(node); + assert(0 <= len && len <= 256); + + if (len == 0) { + // We're at the end of the prefix; make this a leaf and set the + // value. + if (!IS_LEAF(node)) { + _convert_to_leaf(node); + } + node->value = value; + return; + } + + if (IS_LEAF(node)) { + // We're not at the end of the prefix, but we hit a leaf. + if (node->value == value) { + // A larger prefix has the same value, so we're done. + return; + } + // The larger prefix has a different value, so we need to + // convert it into an internal node and continue processing on + // one of the leaves. + node->l = _create_leaf(node->value); + node->r = _create_leaf(node->value); + } + + // We're not at the end of the prefix, and we're at an internal + // node. Recurse on the left or right subtree. + if (prefix & 0x80000000) { + _set_recurse(node->r, prefix << 1, len - 1, value); + } else { + _set_recurse(node->l, prefix << 1, len - 1, value); + } + + // At this point, we're an internal node, and the value is set + // by one of our children or its descendent. If both children are + // leaves with the same value, we can discard them and become a left. + if (IS_LEAF(node->r) && IS_LEAF(node->l) && + node->r->value == node->l->value) { + node->value = node->l->value; + _convert_to_leaf(node); + } +} + +// Set the value for a given network prefix, overwriting any existing +// values on that prefix or subsets of it. +// (Note: prefix must be in host byte order.) +void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value) +{ + assert(con); + _set_recurse(con->root, prefix, len, value); + con->painted = 0; +} + +// Return the value pertaining to an address, according to the tree +// starting at given root. (Note: address must be in host byte order.) +static int _lookup_ip(node_t *root, uint32_t address) +{ + assert(root); + node_t *node = root; + uint32_t mask = 0x80000000; + for (;;) { + if (IS_LEAF(node)) { + return node->value; + } + if (address & mask) { + node = node->r; + } else { + node = node->l; + } + mask >>= 1; + } +} + +// Return the value pertaining to an address. +// (Note: address must be in host byte order.) +value_t constraint_lookup_ip(constraint_t *con, uint32_t address) +{ + assert(con); + return _lookup_ip(con->root, address); +} + +// Return the nth painted IP address. +static int _lookup_index(node_t *root, uint64_t n) +{ + assert(root); + node_t *node = root; + uint32_t ip = 0; + uint32_t mask = 0x80000000; + for (;;) { + if (IS_LEAF(node)) { + return ip | n; + } + if (n < node->l->count) { + node = node->l; + } else { + n -= node->l->count; + node = node->r; + ip |= mask; + } + mask >>= 1; + } +} + +// For a given value, return the IP address with zero-based index n. +// (i.e., if there are three addresses with value 0xFF, looking up index 1 +// will return the second one). +// Note that the tree must have been previously painted with this value. +uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, + value_t value) +{ + assert(con); + if (!con->painted || con->paint_value != value) { + constraint_paint_value(con, value); + } + + uint64_t radix_idx = index / (1 << (32 - RADIX_LENGTH)); + if (radix_idx < con->radix_len) { + // Radix lookup + uint32_t radix_offset = + index % (1 << (32 - RADIX_LENGTH)); // TODO: bitwise maths + return con->radix[radix_idx] | radix_offset; + } + + // Otherwise, do the "slow" lookup in tree. + // Note that tree counts do NOT include things in the radix, + // so we subtract these off here. + index -= con->radix_len * (1 << (32 - RADIX_LENGTH)); + assert(index < con->root->count); + return _lookup_index(con->root, index); +} + +// Implement count_ips by recursing on halves of the tree. Size represents +// the number of addresses in a prefix at the current level of the tree. +// If paint is specified, each node will have its count set to the number of +// leaves under it set to value. +// If exclude_radix is specified, the number of addresses will exclude prefixes +// that are a /RADIX_LENGTH or larger +static uint64_t _count_ips_recurse(node_t *node, value_t value, uint64_t size, + int paint, int exclude_radix) +{ + assert(node); + uint64_t n; + if (IS_LEAF(node)) { + if (node->value == value) { + n = size; + // Exclude prefixes already included in the radix + if (exclude_radix && + size >= (1 << (32 - RADIX_LENGTH))) { + n = 0; + } + } else { + n = 0; + } + } else { + n = _count_ips_recurse(node->l, value, size >> 1, paint, + exclude_radix) + + _count_ips_recurse(node->r, value, size >> 1, paint, + exclude_radix); + } + if (paint) { + node->count = n; + } + return n; +} + +// Return a node that determines the values for the addresses with +// the given prefix. This is either the internal node that +// corresponds to the end of the prefix or a leaf node that +// encompasses the prefix. (Note: prefix must be in host byte order.) +static node_t *_lookup_node(node_t *root, uint32_t prefix, int len) +{ + assert(root); + assert(0 <= len && len <= 32); + + node_t *node = root; + uint32_t mask = 0x80000000; + int i; + + for (i = 0; i < len; i++) { + if (IS_LEAF(node)) { + return node; + } + if (prefix & mask) { + node = node->r; + } else { + node = node->l; + } + mask >>= 1; + } + return node; +} + +// For each node, precompute the count of leaves beneath it set to value. +// Note that the tree can be painted for only one value at a time. +void constraint_paint_value(constraint_t *con, value_t value) +{ + assert(con); + log_debug("constraint", "Painting value %lu", value); + + // Paint everything except what we will put in radix + _count_ips_recurse(con->root, value, (uint64_t)1 << 32, 1, 1); + + // Fill in the radix array with a list of addresses + uint32_t i; + con->radix_len = 0; + for (i = 0; i < (1 << RADIX_LENGTH); i++) { + uint32_t prefix = i << (32 - RADIX_LENGTH); + node_t *node = _lookup_node(con->root, prefix, RADIX_LENGTH); + if (IS_LEAF(node) && node->value == value) { + // Add this prefix to the radix + con->radix[con->radix_len++] = prefix; + } + } + log_debug("constraint", "%lu IPs in radix array, %lu IPs in tree", + con->radix_len * (1 << (32 - RADIX_LENGTH)), + con->root->count); + con->painted = 1; + con->paint_value = value; +} + +// Return the number of addresses that have a given value. +uint64_t constraint_count_ips(constraint_t *con, value_t value) +{ + assert(con); + if (con->painted && con->paint_value == value) { + return con->root->count + + con->radix_len * (1 << (32 - RADIX_LENGTH)); + } else { + return _count_ips_recurse(con->root, value, (uint64_t)1 << 32, + 0, 0); + } +} + +// Initialize the tree. +// All addresses will initially have the given value. +constraint_t *constraint_init(value_t value) +{ + constraint_t *con = xmalloc(sizeof(constraint_t)); + con->root = _create_leaf(value); + con->radix = xcalloc(sizeof(uint32_t), 1 << RADIX_LENGTH); + con->painted = 0; + return con; +} + +// Deinitialize and free the tree. +void constraint_free(constraint_t *con) +{ + assert(con); + log_debug("constraint", "Cleaning up"); + _destroy_subtree(con->root); + free(con->radix); + free(con); +} + +/* +int main(void) +{ + log_init(stderr, LOG_DEBUG); + + constraint_t *con = constraint_init(0); + constraint_set(con, ntohl(inet_addr("128.128.0.0")), 1, 22); + constraint_set(con, ntohl(inet_addr("128.128.0.0")), 1, 1); + constraint_set(con, ntohl(inet_addr("128.0.0.0")), 1, 1); + constraint_set(con, ntohl(inet_addr("10.0.0.0")), 24, 1); + constraint_set(con, ntohl(inet_addr("10.0.0.0")), 24, 0); + constraint_set(con, ntohl(inet_addr("10.11.12.0")), 24, 1); + constraint_set(con, ntohl(inet_addr("141.212.0.0")), 16, 0); + + for (int x=1; x < 2; x++) { + if (x == 1) { + constraint_optimize(con); + } + + printf("count(0)=%ld\n", constraint_count_ips(con, 0)); + printf("count(1)=%ld\n", constraint_count_ips(con, 1)); + printf("%d\n", +constraint_lookup_ip(con,ntohl(inet_addr("10.11.12.0")))); + assert(constraint_count_ips(con, 0) + constraint_count_ips(con, +1) == (uint64_t)1 << 32); + + uint32_t i=0, count=0; + do { + if (constraint_lookup_ip(con, i)) + count++; + } while (++i != 0); + printf("derived count(1)=%u\n", count); + } + + constraint_free(con); +} +*/ diff --git a/rdns_scan/zmap4rdns/lib/constraint.h b/rdns_scan/zmap4rdns/lib/constraint.h new file mode 100644 index 0000000..57e9b88 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/constraint.h @@ -0,0 +1,34 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONSTRAINT_H +#define CONSTRAINT_H + +#include <stdint.h> + +typedef struct _constraint constraint_t; +typedef uint32_t value_t; + +constraint_t *constraint_init(value_t value); +void constraint_free(constraint_t *con); +void constraint_set(constraint_t *con, uint32_t prefix, int len, value_t value); +value_t constraint_lookup_ip(constraint_t *con, uint32_t address); +uint64_t constraint_count_ips(constraint_t *con, value_t value); +uint32_t constraint_lookup_index(constraint_t *con, uint64_t index, + value_t value); +void constraint_paint_value(constraint_t *con, value_t value); + +#endif //_CONSTRAINT_H diff --git a/rdns_scan/zmap4rdns/lib/csv.c b/rdns_scan/zmap4rdns/lib/csv.c new file mode 100644 index 0000000..190f6f3 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/csv.c @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "csv.h" + +int csv_find_index(char *header, const char **names, size_t names_len) +{ + char *split = header; + for (int idx = 0; split != NULL; ++idx) { + char *front = (idx == 0) ? split : split + 1; + for (size_t i = 0; i < names_len; ++i) { + if (strncmp(front, names[i], strlen(names[i])) == 0) { + return idx; + } + } + split = strchr(front, ','); + } + return -1; +} + +char *csv_get_index(char *row, size_t idx) +{ + char *split = row; + for (size_t i = 0; i < idx; ++i) { + split = strchr(split + 1, ','); + if (split == NULL) { + return NULL; + } + } + char *entry; + char *start = (idx == 0) ? split : split + 1; + char *end = strchr(start, ','); + if (end != NULL) { + entry = strndup(start, end - start); + } else { + entry = strdup(start); + } + return entry; +} diff --git a/rdns_scan/zmap4rdns/lib/csv.h b/rdns_scan/zmap4rdns/lib/csv.h new file mode 100644 index 0000000..9229cdc --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/csv.h @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_CSV_H +#define ZMAP_CSV_H + +#include <string.h> + +int csv_find_index(char *header, const char **names, size_t names_len); +char *csv_get_index(char *row, size_t idx); + +#endif diff --git a/rdns_scan/zmap4rdns/lib/includes.h b/rdns_scan/zmap4rdns/lib/includes.h new file mode 100644 index 0000000..1bf6bad --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/includes.h @@ -0,0 +1,51 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __APPLE__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#include <dnet.h> +#pragma GCC diagnostic warning "-Wflexible-array-extensions" +#endif + +#ifndef __FAVOR_BSD +#define __FAVOR_BSD 2 +#endif +#ifndef __USE_BSD +#define __USE_BSD +#endif + +#include <sys/types.h> +#include <netinet/in.h> +#include <netinet/ip.h> +#include <netinet/ip_icmp.h> +#include <netinet/udp.h> +#include <netinet/tcp.h> +#include <netinet/ip_icmp.h> + +#if defined(__NetBSD__) +#define ICMP_UNREACH_PRECEDENCE_CUTOFF ICMP_UNREACH_PREC_CUTOFF +#include <net/if_ether.h> +#else +#include <net/ethernet.h> +#endif + +#include <netdb.h> +#include <net/if.h> +#include <ifaddrs.h> // NOTE: net/if.h MUST be included BEFORE ifaddrs.h +#include <arpa/inet.h> + +#define MAC_ADDR_LEN ETHER_ADDR_LEN +#define UNUSED __attribute__((unused)) diff --git a/rdns_scan/zmap4rdns/lib/lockfd.c b/rdns_scan/zmap4rdns/lib/lockfd.c new file mode 100644 index 0000000..093e819 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/lockfd.c @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <pthread.h> +#include <assert.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> + +#include "xalloc.h" + +static pthread_mutex_t **mutexes = NULL; + +static pthread_mutex_t *get_mutex(int fd) +{ + assert(fd < 3 && "todo: implement generically"); + if (!mutexes) { + mutexes = xmalloc(3 * sizeof(char *)); + assert(mutexes); + } + if (!mutexes[fd]) { + mutexes[fd] = xmalloc(sizeof(pthread_mutex_t)); + assert(mutexes[fd]); + pthread_mutex_init(mutexes[fd], NULL); + assert(mutexes[fd]); + } + return mutexes[fd]; +} + +int lock_fd(int fd) { return pthread_mutex_lock(get_mutex(fd)); } + +int unlock_fd(int fd) { return pthread_mutex_unlock(get_mutex(fd)); } + +int lock_file(FILE *f) +{ + assert(f); + return lock_fd(fileno(f)); +} + +int unlock_file(FILE *f) +{ + assert(f); + return unlock_fd(fileno(f)); +} diff --git a/rdns_scan/zmap4rdns/lib/lockfd.h b/rdns_scan/zmap4rdns/lib/lockfd.h new file mode 100644 index 0000000..625534a --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/lockfd.h @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> + +int lock_fd(int fd); +int unlock_fd(int fd); +int lock_file(FILE *f); +int unlock_file(FILE *f); diff --git a/rdns_scan/zmap4rdns/lib/logger.c b/rdns_scan/zmap4rdns/lib/logger.c new file mode 100644 index 0000000..e995659 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/logger.c @@ -0,0 +1,278 @@ +/* + * Logger Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <unistd.h> +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <sys/time.h> +#include <time.h> +#include <syslog.h> +#include <math.h> +#include <pthread.h> + +#include "logger.h" +#include "xalloc.h" +#include "lockfd.h" + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static enum LogLevel log_output_level = ZLOG_INFO; + +static FILE *log_output_stream = NULL; +static int color = 0; + +static int log_to_syslog = 0; + +static const char *log_level_name[] = {"FATAL", "ERROR", "WARN", + "INFO", "DEBUG", "TRACE"}; + +#define RED "\x1b[31m" +#define GREEN "\x1b[32m" +#define YELLOW "\x1b[33m" +#define BLUE "\x1b[34m" +#define MAGENTA "\x1b[35m" +#define CYAN "\x1b[36m" +#define RESET "\033[0m" + +#define COLOR(x) \ + do { \ + if (color) \ + fprintf(log_output_stream, "%s", x); \ + } while (0) + +static const char *color_for_level(enum LogLevel level) +{ + switch (level) { + case ZLOG_FATAL: + return RED; + case ZLOG_ERROR: + return MAGENTA; + case ZLOG_WARN: + return YELLOW; + case ZLOG_INFO: + return GREEN; + case ZLOG_DEBUG: + return BLUE; + case ZLOG_TRACE: + return RESET; + default: + return RESET; + } +} + +static int LogLogVA(enum LogLevel level, const char *loggerName, + const char *logMessage, va_list args) +{ + if (level <= log_output_level) { + if (!log_output_stream) { + log_output_stream = stderr; + } + // if logging to a shared output channel, then use a global + // lock across ZMap. Otherwise, if we're logging to a file, + // only lockin with the module, in order to avoid having + // corrupt log entries. + if (log_output_stream == stdout || + log_output_stream == stderr) { + lock_file(log_output_stream); + } else { + pthread_mutex_lock(&mutex); + } + if (color) { + COLOR(color_for_level(level)); + } + + const char *levelName = log_level_name[level]; + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%b %d %H:%M:%S", ptm); + fprintf(log_output_stream, "%s.%03ld [%s] ", timestamp, + (long)now.tv_usec / 1000, levelName); + if (loggerName) { + fprintf(log_output_stream, "%s: ", loggerName); + } + if (logMessage) { + vfprintf(log_output_stream, logMessage, args); + } + if (loggerName || logMessage) { + fputs("\n", log_output_stream); + } + if (color) { + COLOR(RESET); + } + fflush(log_output_stream); + if (log_output_stream == stdout || + log_output_stream == stderr) { + unlock_file(log_output_stream); + } else { + pthread_mutex_unlock(&mutex); + } + } + return EXIT_SUCCESS; +} + +int log_fatal(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + LogLogVA(ZLOG_FATAL, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_CRIT), message, va); + va_end(va); + } + + exit(EXIT_FAILURE); +} + +int log_error(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_ERROR, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_ERR), message, va); + va_end(va); + } + + return ret; +} + +int log_warn(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_WARN, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_WARNING), message, va); + va_end(va); + } + + return ret; +} + +int log_info(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_INFO, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_INFO), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +int log_debug(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_DEBUG, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +#ifdef DEBUG +extern int log_trace(const char *name, const char *message, ...) +{ + va_list va; + va_start(va, message); + int ret = LogLogVA(ZLOG_TRACE, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *appname) +{ + log_output_stream = stream; + log_output_level = level; + if (syslog_enabled) { + log_to_syslog = 1; + openlog(appname, 0, LOG_USER); // no options + } + if (isatty(fileno(log_output_stream))) { + color = 1; + } + return 0; +} + +void check_and_log_file_error(FILE *file, const char *name) +{ + if (ferror(file)) { + log_fatal(name, "unable to write to file"); + } +} + +double now(void) +{ + struct timeval now; + gettimeofday(&now, NULL); + return (double)now.tv_sec + (double)now.tv_usec / 1000000.; +} + +size_t dstrftime(char *buf, size_t maxsize, const char *format, double tm) +{ + struct timeval tv; + double tm_floor; + tm_floor = floor(tm); + tv.tv_sec = (long)tm_floor; + tv.tv_usec = (long)(tm - floor(tm)) * 1000000; + return strftime(buf, maxsize, format, localtime((const time_t *)&tv)); +} diff --git a/rdns_scan/zmap4rdns/lib/logger.h b/rdns_scan/zmap4rdns/lib/logger.h new file mode 100644 index 0000000..6a80ca6 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/logger.h @@ -0,0 +1,69 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <stdarg.h> +#include <syslog.h> + +#ifndef LOGGER_H +#define LOGGER_H + +#ifdef __cplusplus +extern "C" { +#endif + +// do not collide with constants defined in syslog.h +enum LogLevel { + ZLOG_FATAL, + ZLOG_ERROR, + ZLOG_WARN, + ZLOG_INFO, + ZLOG_DEBUG, + ZLOG_TRACE, + ZNUM_LOGLEVELS +}; + +int log_fatal(const char *loggerName, const char *logMessage, ...) + __attribute__((noreturn)); + +int log_error(const char *loggerName, const char *logMessage, ...); + +int log_warn(const char *loggerName, const char *logMessage, ...); + +int log_info(const char *loggerName, const char *logMessage, ...); + +int log_debug(const char *loggerName, const char *logMessage, ...); + +#ifdef DEBUG +int log_trace(const char *loggerName, const char *logMessage, ...); +#else +#define log_trace(...) ; +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *syslog_app); + +void check_and_log_file_error(FILE *file, const char *name); + +size_t dstrftime(char *, size_t, const char *, double); + +double now(); + +#ifdef __cplusplus +} +#endif + +#endif // _LOGGER_H diff --git a/rdns_scan/zmap4rdns/lib/pbm.c b/rdns_scan/zmap4rdns/lib/pbm.c new file mode 100644 index 0000000..b884f4f --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/pbm.c @@ -0,0 +1,106 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <assert.h> +#include <errno.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <stdio.h> + +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "logger.h" +#include "xalloc.h" + +#define NUM_VALUES 0xFFFFFFFF +#define PAGE_SIZE_IN_BITS 0x10000 +#define PAGE_SIZE_IN_BYTES (PAGE_SIZE_IN_BITS / 8) +#define NUM_PAGES 0x10000 +#define PAGE_MASK 0xFFFF + +uint8_t **pbm_init(void) +{ + uint8_t **retv = xcalloc(NUM_PAGES, sizeof(void *)); + return retv; +} + +static inline int bm_check(uint8_t *bm, uint16_t v) +{ + uint16_t page_idx = (v >> 3); + uint8_t bit_idx = (uint8_t)(v & 0x07); + return bm[page_idx] & (1 << bit_idx); +} + +static inline void bm_set(uint8_t *bm, uint16_t v) +{ + uint16_t page_idx = (v >> 3); + uint8_t bit_idx = (uint8_t)(v & 0x07); + bm[page_idx] |= (1 << bit_idx); +} + +int pbm_check(uint8_t **b, uint32_t v) +{ + uint32_t top = v >> 16; + uint32_t bottom = v & PAGE_MASK; + return b[top] && bm_check(b[top], bottom); +} + +void pbm_set(uint8_t **b, uint32_t v) +{ + uint16_t top = (uint16_t)(v >> 16); + uint16_t bottom = (uint16_t)(v & PAGE_MASK); + if (!b[top]) { + uint8_t *bm = xmalloc(PAGE_SIZE_IN_BYTES); + memset(bm, 0, PAGE_SIZE_IN_BYTES); + b[top] = bm; + } + bm_set(b[top], bottom); +} + +uint32_t pbm_load_from_file(uint8_t **b, char *file) +{ + if (!b) { + log_fatal("pbm", "load_from_file called with NULL PBM"); + } + if (!file) { + log_fatal("pbm", "load_from_file called with NULL filename"); + } + FILE *fp = fopen(file, "r"); + if (fp == NULL) { + log_fatal("pbm", "unable to open file: %s: %s", file, + strerror(errno)); + } + char line[1000]; + uint32_t count = 0; + while (fgets(line, sizeof(line), fp)) { + char *comment = strchr(line, '#'); + if (comment) { + *comment = '\0'; + } + struct in_addr addr; + if (inet_aton(line, &addr) != 1) { + log_fatal("pbm", "unable to parse IP address: %s", + line); + } + pbm_set(b, addr.s_addr); + ++count; + } + fclose(fp); + return count; +} diff --git a/rdns_scan/zmap4rdns/lib/pbm.h b/rdns_scan/zmap4rdns/lib/pbm.h new file mode 100644 index 0000000..051975e --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/pbm.h @@ -0,0 +1,27 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_PBM_H +#define ZMAP_PBM_H + +#include <stdint.h> + +uint8_t **pbm_init(void); +int pbm_check(uint8_t **b, uint32_t v); +void pbm_set(uint8_t **b, uint32_t v); +uint32_t pbm_load_from_file(uint8_t **b, char *file); + +#endif /* ZMAP_PBM_H */ diff --git a/rdns_scan/zmap4rdns/lib/queue.c b/rdns_scan/zmap4rdns/lib/queue.c new file mode 100644 index 0000000..bf91d78 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/queue.c @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "queue.h" + +#include "xalloc.h" + +#include <pthread.h> + +zqueue_t *queue_init() +{ + zqueue_t *p = xmalloc(sizeof(zqueue_t)); + p->front = NULL; + p->back = NULL; + p->size = 0; + + pthread_mutex_init(&p->lock, NULL); + pthread_cond_init(&p->empty, NULL); + return p; +} + +int is_empty(zqueue_t *queue) { return queue->size == 0; } + +void push_back(char *data, zqueue_t *queue) +{ + znode_t *new_node = xmalloc(sizeof(znode_t)); + new_node->prev = NULL; + new_node->next = NULL; + new_node->data = strdup(data); + + pthread_mutex_lock(&queue->lock); + if (is_empty(queue)) { + queue->front = new_node; + queue->back = new_node; + } else { + queue->back->next = new_node; + new_node->prev = queue->back; + queue->back = new_node; + } + queue->size++; + pthread_cond_signal(&queue->empty); + pthread_mutex_unlock(&queue->lock); +} + +znode_t *pop_front(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + znode_t *temp = pop_front_unsafe(queue); + pthread_mutex_unlock(&queue->lock); + return temp; +} + +znode_t *pop_front_unsafe(zqueue_t *queue) +{ + znode_t *temp = queue->front; + queue->front = temp->next; + if (queue->front != NULL) { + queue->front->prev = NULL; + } + queue->size--; + return temp; +} + +znode_t *get_front(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + + znode_t *temp = xmalloc(sizeof(znode_t)); + temp = queue->front; + pthread_mutex_unlock(&queue->lock); + return temp; +} + +znode_t *get_back(zqueue_t *queue) +{ + pthread_mutex_lock(&queue->lock); + + while (is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + + znode_t *temp = xmalloc(sizeof(znode_t)); + temp = queue->back; + pthread_mutex_unlock(&queue->lock); + return temp; +} + +size_t get_size(zqueue_t *queue) { return queue->size; } diff --git a/rdns_scan/zmap4rdns/lib/queue.h b/rdns_scan/zmap4rdns/lib/queue.h new file mode 100644 index 0000000..da7f218 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/queue.h @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_QUEUE_H +#define ZMAP_QUEUE_H + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> + +typedef struct zqueue_node { + char *data; + struct zqueue_node *prev; + struct zqueue_node *next; +} znode_t; + +typedef struct zqueue { + struct zqueue_node *front; + struct zqueue_node *back; + size_t size; + // Threading utilities + pthread_mutex_t lock; + pthread_cond_t empty; +} zqueue_t; + +zqueue_t *queue_init(); +int is_empty(zqueue_t *queue); +void push_back(char *data, zqueue_t *queue); +znode_t *pop_front(zqueue_t *queue); +znode_t *pop_front_unsafe(zqueue_t *queue); +znode_t *get_front(zqueue_t *queue); +znode_t *get_back(zqueue_t *queue); +size_t get_size(zqueue_t *queue); + +#endif /* ZMAP_QUEUE_H */ diff --git a/rdns_scan/zmap4rdns/lib/random.c b/rdns_scan/zmap4rdns/lib/random.c new file mode 100644 index 0000000..e3ac8f8 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/random.c @@ -0,0 +1,32 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "random.h" +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <errno.h> +#include "logger.h" + +#define RANDSRC "/dev/urandom" + +int random_bytes(void *dst, size_t n) +{ + FILE *f = fopen(RANDSRC, "rb"); + if (!f) { + log_fatal("random", "unable to read /dev/urandom: %s", + strerror(errno)); + } + size_t r = fread(dst, n, 1, f); + fclose(f); + if (r < 1) { + return 0; + } + return 1; +} diff --git a/rdns_scan/zmap4rdns/lib/random.h b/rdns_scan/zmap4rdns/lib/random.h new file mode 100644 index 0000000..1151efe --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/random.h @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> +#include <stdint.h> + +#ifndef RANDOM_H +#define RANDOM_H + +int random_bytes(void *dst, size_t n); + +#endif diff --git a/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.c b/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.c new file mode 100644 index 0000000..4a36327 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.c @@ -0,0 +1,1284 @@ +/** + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen <[email protected]> + * @author Antoon Bosselaers <[email protected]> + * @author Paulo Barreto <[email protected]> + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <assert.h> +#include <stdlib.h> + +#include "rijndael-alg-fst.h" + +/* +Te0[x] = S [x].[02, 01, 01, 03]; +Te1[x] = S [x].[03, 02, 01, 01]; +Te2[x] = S [x].[01, 03, 02, 01]; +Te3[x] = S [x].[01, 01, 03, 02]; +Te4[x] = S [x].[01, 01, 01, 01]; + +Td0[x] = Si[x].[0e, 09, 0d, 0b]; +Td1[x] = Si[x].[0b, 0e, 09, 0d]; +Td2[x] = Si[x].[0d, 0b, 0e, 09]; +Td3[x] = Si[x].[09, 0d, 0b, 0e]; +Td4[x] = Si[x].[01, 01, 01, 01]; +*/ + +static const u32 Te0[256] = { + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, 0xfff2f20dU, + 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, 0x60303050U, 0x02010103U, + 0xce6767a9U, 0x562b2b7dU, 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, + 0xec76769aU, 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, 0x41adadecU, + 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, 0x239c9cbfU, 0x53a4a4f7U, + 0xe4727296U, 0x9bc0c05bU, 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, + 0x4c26266aU, 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, 0xe2717193U, + 0xabd8d873U, 0x62313153U, 0x2a15153fU, 0x0804040cU, 0x95c7c752U, + 0x46232365U, 0x9dc3c35eU, 0x30181828U, 0x379696a1U, 0x0a05050fU, + 0x2f9a9ab5U, 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, 0x1209091bU, + 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, 0x361b1b2dU, 0xdc6e6eb2U, + 0xb45a5aeeU, 0x5ba0a0fbU, 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, + 0x7db3b3ceU, 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, 0x40202060U, + 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, 0xd46a6abeU, 0x8dcbcb46U, + 0x67bebed9U, 0x7239394bU, 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, + 0x85cfcf4aU, 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, 0x8a4545cfU, + 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, 0xa05050f0U, 0x783c3c44U, + 0x259f9fbaU, 0x4ba8a8e3U, 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, + 0x058f8f8aU, 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, 0x20101030U, + 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, 0x81cdcd4cU, 0x180c0c14U, + 0x26131335U, 0xc3ecec2fU, 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, + 0x2e171739U, 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, 0xc06060a0U, + 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, 0x44222266U, 0x542a2a7eU, + 0x3b9090abU, 0x0b888883U, 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, + 0x2814143cU, 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, 0x924949dbU, + 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, 0x9fc2c25dU, 0xbdd3d36eU, + 0x43acacefU, 0xc46262a6U, 0x399191a8U, 0x319595a4U, 0xd3e4e437U, + 0xf279798bU, 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, 0xd86c6cb4U, + 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, 0xca6565afU, 0xf47a7a8eU, + 0x47aeaee9U, 0x10080818U, 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, + 0x5c2e2e72U, 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, 0x964b4bddU, + 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, 0xe0707090U, 0x7c3e3e42U, + 0x71b5b5c4U, 0xcc6666aaU, 0x904848d8U, 0x06030305U, 0xf7f6f601U, + 0x1c0e0e12U, 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, 0xd9e1e138U, + 0xebf8f813U, 0x2b9898b3U, 0x22111133U, 0xd26969bbU, 0xa9d9d970U, + 0x078e8e89U, 0x339494a7U, 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, + 0xc9e9e920U, 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, 0x65bfbfdaU, + 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, 0x824141c3U, 0x299999b0U, + 0x5a2d2d77U, 0x1e0f0f11U, 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, + 0x2c16163aU, +}; +static const u32 Te1[256] = { + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, 0x0dfff2f2U, + 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, 0x50603030U, 0x03020101U, + 0xa9ce6767U, 0x7d562b2bU, 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, + 0x9aec7676U, 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, 0xec41adadU, + 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, 0xbf239c9cU, 0xf753a4a4U, + 0x96e47272U, 0x5b9bc0c0U, 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, + 0x6a4c2626U, 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, 0x93e27171U, + 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, 0x0c080404U, 0x5295c7c7U, + 0x65462323U, 0x5e9dc3c3U, 0x28301818U, 0xa1379696U, 0x0f0a0505U, + 0xb52f9a9aU, 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, 0x1b120909U, + 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, 0x2d361b1bU, 0xb2dc6e6eU, + 0xeeb45a5aU, 0xfb5ba0a0U, 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, + 0xce7db3b3U, 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, 0x60402020U, + 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, 0xbed46a6aU, 0x468dcbcbU, + 0xd967bebeU, 0x4b723939U, 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, + 0x4a85cfcfU, 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, 0xcf8a4545U, + 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, 0xf0a05050U, 0x44783c3cU, + 0xba259f9fU, 0xe34ba8a8U, 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, + 0x8a058f8fU, 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, 0x30201010U, + 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, 0x4c81cdcdU, 0x14180c0cU, + 0x35261313U, 0x2fc3ececU, 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, + 0x392e1717U, 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, 0xa0c06060U, + 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, 0x66442222U, 0x7e542a2aU, + 0xab3b9090U, 0x830b8888U, 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, + 0x3c281414U, 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, 0xdb924949U, + 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, 0x5d9fc2c2U, 0x6ebdd3d3U, + 0xef43acacU, 0xa6c46262U, 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, + 0x8bf27979U, 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, 0xb4d86c6cU, + 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, 0xafca6565U, 0x8ef47a7aU, + 0xe947aeaeU, 0x18100808U, 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, + 0x725c2e2eU, 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, 0xdd964b4bU, + 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, 0x90e07070U, 0x427c3e3eU, + 0xc471b5b5U, 0xaacc6666U, 0xd8904848U, 0x05060303U, 0x01f7f6f6U, + 0x121c0e0eU, 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, 0x38d9e1e1U, + 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, 0xbbd26969U, 0x70a9d9d9U, + 0x89078e8eU, 0xa7339494U, 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, + 0x20c9e9e9U, 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, 0xda65bfbfU, + 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, 0xc3824141U, 0xb0299999U, + 0x775a2d2dU, 0x111e0f0fU, 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, + 0x3a2c1616U, +}; +static const u32 Te2[256] = { + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, 0xf20dfff2U, + 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, 0x30506030U, 0x01030201U, + 0x67a9ce67U, 0x2b7d562bU, 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, + 0x769aec76U, 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, 0xadec41adU, + 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, 0x9cbf239cU, 0xa4f753a4U, + 0x7296e472U, 0xc05b9bc0U, 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, + 0x266a4c26U, 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, 0x7193e271U, + 0xd873abd8U, 0x31536231U, 0x153f2a15U, 0x040c0804U, 0xc75295c7U, + 0x23654623U, 0xc35e9dc3U, 0x18283018U, 0x96a13796U, 0x050f0a05U, + 0x9ab52f9aU, 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, 0x091b1209U, + 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, 0x1b2d361bU, 0x6eb2dc6eU, + 0x5aeeb45aU, 0xa0fb5ba0U, 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, + 0xb3ce7db3U, 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, 0x20604020U, + 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, 0x6abed46aU, 0xcb468dcbU, + 0xbed967beU, 0x394b7239U, 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, + 0xcf4a85cfU, 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, 0x45cf8a45U, + 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, 0x50f0a050U, 0x3c44783cU, + 0x9fba259fU, 0xa8e34ba8U, 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, + 0x8f8a058fU, 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, 0x10302010U, + 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, 0xcd4c81cdU, 0x0c14180cU, + 0x13352613U, 0xec2fc3ecU, 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, + 0x17392e17U, 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, 0x60a0c060U, + 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, 0x22664422U, 0x2a7e542aU, + 0x90ab3b90U, 0x88830b88U, 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, + 0x143c2814U, 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, 0x49db9249U, + 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, 0xc25d9fc2U, 0xd36ebdd3U, + 0xacef43acU, 0x62a6c462U, 0x91a83991U, 0x95a43195U, 0xe437d3e4U, + 0x798bf279U, 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, 0x6cb4d86cU, + 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, 0x65afca65U, 0x7a8ef47aU, + 0xaee947aeU, 0x08181008U, 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, + 0x2e725c2eU, 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, 0x4bdd964bU, + 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, 0x7090e070U, 0x3e427c3eU, + 0xb5c471b5U, 0x66aacc66U, 0x48d89048U, 0x03050603U, 0xf601f7f6U, + 0x0e121c0eU, 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, 0xe138d9e1U, + 0xf813ebf8U, 0x98b32b98U, 0x11332211U, 0x69bbd269U, 0xd970a9d9U, + 0x8e89078eU, 0x94a73394U, 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, + 0xe920c9e9U, 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, 0xbfda65bfU, + 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, 0x41c38241U, 0x99b02999U, + 0x2d775a2dU, 0x0f111e0fU, 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, + 0x163a2c16U, +}; +static const u32 Te3[256] = { + + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, 0xf2f20dffU, + 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, 0x30305060U, 0x01010302U, + 0x6767a9ceU, 0x2b2b7d56U, 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, + 0x76769aecU, 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, 0xadadec41U, + 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, 0x9c9cbf23U, 0xa4a4f753U, + 0x727296e4U, 0xc0c05b9bU, 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, + 0x26266a4cU, 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, 0x717193e2U, + 0xd8d873abU, 0x31315362U, 0x15153f2aU, 0x04040c08U, 0xc7c75295U, + 0x23236546U, 0xc3c35e9dU, 0x18182830U, 0x9696a137U, 0x05050f0aU, + 0x9a9ab52fU, 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, 0x09091b12U, + 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, 0x1b1b2d36U, 0x6e6eb2dcU, + 0x5a5aeeb4U, 0xa0a0fb5bU, 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, + 0xb3b3ce7dU, 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, 0x20206040U, + 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, 0x6a6abed4U, 0xcbcb468dU, + 0xbebed967U, 0x39394b72U, 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, + 0xcfcf4a85U, 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, 0x4545cf8aU, + 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, 0x5050f0a0U, 0x3c3c4478U, + 0x9f9fba25U, 0xa8a8e34bU, 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, + 0x8f8f8a05U, 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, 0x10103020U, + 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, 0xcdcd4c81U, 0x0c0c1418U, + 0x13133526U, 0xecec2fc3U, 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, + 0x1717392eU, 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, 0x6060a0c0U, + 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, 0x22226644U, 0x2a2a7e54U, + 0x9090ab3bU, 0x8888830bU, 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, + 0x14143c28U, 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, 0x4949db92U, + 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, 0xc2c25d9fU, 0xd3d36ebdU, + 0xacacef43U, 0x6262a6c4U, 0x9191a839U, 0x9595a431U, 0xe4e437d3U, + 0x79798bf2U, 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, 0x6c6cb4d8U, + 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, 0x6565afcaU, 0x7a7a8ef4U, + 0xaeaee947U, 0x08081810U, 0xbabad56fU, 0x787888f0U, 0x25256f4aU, + 0x2e2e725cU, 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, 0x4b4bdd96U, + 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, 0x707090e0U, 0x3e3e427cU, + 0xb5b5c471U, 0x6666aaccU, 0x4848d890U, 0x03030506U, 0xf6f601f7U, + 0x0e0e121cU, 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, 0xe1e138d9U, + 0xf8f813ebU, 0x9898b32bU, 0x11113322U, 0x6969bbd2U, 0xd9d970a9U, + 0x8e8e8907U, 0x9494a733U, 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, + 0xe9e920c9U, 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, 0xbfbfda65U, + 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, 0x4141c382U, 0x9999b029U, + 0x2d2d775aU, 0x0f0f111eU, 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, + 0x16163a2cU, +}; +static const u32 Te4[256] = { + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, 0xf2f2f2f2U, + 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, 0x30303030U, 0x01010101U, + 0x67676767U, 0x2b2b2b2bU, 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, + 0x76767676U, 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, 0xadadadadU, + 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, 0x9c9c9c9cU, 0xa4a4a4a4U, + 0x72727272U, 0xc0c0c0c0U, 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, + 0x26262626U, 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, 0x71717171U, + 0xd8d8d8d8U, 0x31313131U, 0x15151515U, 0x04040404U, 0xc7c7c7c7U, + 0x23232323U, 0xc3c3c3c3U, 0x18181818U, 0x96969696U, 0x05050505U, + 0x9a9a9a9aU, 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, 0x09090909U, + 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, 0x1b1b1b1bU, 0x6e6e6e6eU, + 0x5a5a5a5aU, 0xa0a0a0a0U, 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, + 0xb3b3b3b3U, 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, 0x20202020U, + 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, 0x6a6a6a6aU, 0xcbcbcbcbU, + 0xbebebebeU, 0x39393939U, 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, + 0xcfcfcfcfU, 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, 0x45454545U, + 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, 0x50505050U, 0x3c3c3c3cU, + 0x9f9f9f9fU, 0xa8a8a8a8U, 0x51515151U, 0xa3a3a3a3U, 0x40404040U, + 0x8f8f8f8fU, 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, 0x10101010U, + 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, 0xcdcdcdcdU, 0x0c0c0c0cU, + 0x13131313U, 0xececececU, 0x5f5f5f5fU, 0x97979797U, 0x44444444U, + 0x17171717U, 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, 0x60606060U, + 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, 0x22222222U, 0x2a2a2a2aU, + 0x90909090U, 0x88888888U, 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, + 0x14141414U, 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, 0x49494949U, + 0x06060606U, 0x24242424U, 0x5c5c5c5cU, 0xc2c2c2c2U, 0xd3d3d3d3U, + 0xacacacacU, 0x62626262U, 0x91919191U, 0x95959595U, 0xe4e4e4e4U, + 0x79797979U, 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, 0x6c6c6c6cU, + 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, 0x65656565U, 0x7a7a7a7aU, + 0xaeaeaeaeU, 0x08080808U, 0xbabababaU, 0x78787878U, 0x25252525U, + 0x2e2e2e2eU, 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, 0x4b4b4b4bU, + 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, 0x70707070U, 0x3e3e3e3eU, + 0xb5b5b5b5U, 0x66666666U, 0x48484848U, 0x03030303U, 0xf6f6f6f6U, + 0x0e0e0e0eU, 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, 0xe1e1e1e1U, + 0xf8f8f8f8U, 0x98989898U, 0x11111111U, 0x69696969U, 0xd9d9d9d9U, + 0x8e8e8e8eU, 0x94949494U, 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, + 0xe9e9e9e9U, 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, 0xbfbfbfbfU, + 0xe6e6e6e6U, 0x42424242U, 0x68686868U, 0x41414141U, 0x99999999U, + 0x2d2d2d2dU, 0x0f0f0f0fU, 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, + 0x16161616U, +}; +static const u32 Td0[256] = { + 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, 0x3bab6bcbU, + 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, 0x2030fa55U, 0xad766df6U, + 0x88cc7691U, 0xf5024c25U, 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, + 0xb562a38fU, 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, + 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, 0x038f5fe7U, + 0x15929c95U, 0xbf6d7aebU, 0x955259daU, 0xd4be832dU, 0x587421d3U, + 0x49e06929U, 0x8ec9c844U, 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, + 0x27b971ddU, 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, + 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, 0xb16477e0U, + 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, 0x70486858U, 0x8f45fd19U, + 0x94de6c87U, 0x527bf8b7U, 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, + 0x6655ab2aU, 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, + 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, 0x8acf1c2bU, + 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, 0x65daf4cdU, 0x0605bed5U, + 0xd134621fU, 0xc4a6fe8aU, 0x342e539dU, 0xa2f355a0U, 0x058ae132U, + 0xa4f6eb75U, 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, + 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, 0x91548db5U, + 0x71c45d05U, 0x0406d46fU, 0x605015ffU, 0x1998fb24U, 0xd6bde997U, + 0x894043ccU, 0x67d99e77U, 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, + 0x79c8eedbU, 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, + 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, 0xfd0efffbU, + 0x0f853856U, 0x3daed51eU, 0x362d3927U, 0x0a0fd964U, 0x685ca621U, + 0x9b5b54d1U, 0x24362e3aU, 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, + 0x1b9b919eU, 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, + 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, 0x0e090d0bU, + 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, 0x57f11985U, 0xaf75074cU, + 0xee99ddbbU, 0xa37f60fdU, 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, + 0x5bfb7e34U, 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, + 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, 0x854a247dU, + 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, 0x1d9e2f4bU, 0xdcb230f3U, + 0x0d8652ecU, 0x77c1e3d0U, 0x2bb3166cU, 0xa970b999U, 0x119448faU, + 0x47e96422U, 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, + 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, 0xa6f581cfU, + 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, 0x2c3a9de4U, 0x5078920dU, + 0x6a5fcc9bU, 0x547e4662U, 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, + 0x82c3aff5U, 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, + 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, 0xcd267809U, + 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, 0xe6956e65U, 0xaaffe67eU, + 0x21bccf08U, 0xef15e8e6U, 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, + 0x29b07cd6U, 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, + 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, 0xf104984aU, + 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, 0x764dd68dU, 0x43efb04dU, + 0xccaa4d54U, 0xe49604dfU, 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, + 0x4665517fU, 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, + 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, 0x9ad7618cU, + 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, 0xcea927eeU, 0xb761c935U, + 0xe11ce5edU, 0x7a47b13cU, 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, + 0x73c737bfU, 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, + 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, 0x161dc372U, + 0xbce2250cU, 0x283c498bU, 0xff0d9541U, 0x39a80171U, 0x080cb3deU, + 0xd8b4e49cU, 0x6456c190U, 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, + 0xd0b85742U, +}; +static const u32 Td1[256] = { + 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, 0xcb3bab6bU, + 0xf11f9d45U, 0xabacfa58U, 0x934be303U, 0x552030faU, 0xf6ad766dU, + 0x9188cc76U, 0x25f5024cU, 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, + 0x8fb562a3U, 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, + 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, 0xe7038f5fU, + 0x9515929cU, 0xebbf6d7aU, 0xda955259U, 0x2dd4be83U, 0xd3587421U, + 0x2949e069U, 0x448ec9c8U, 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, + 0xdd27b971U, 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, + 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, 0xe0b16477U, + 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, 0x58704868U, 0x198f45fdU, + 0x8794de6cU, 0xb7527bf8U, 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, + 0x2a6655abU, 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, + 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, 0x2b8acf1cU, + 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, 0xcd65daf4U, 0xd50605beU, + 0x1fd13462U, 0x8ac4a6feU, 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, + 0x75a4f6ebU, 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, + 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, 0xb591548dU, + 0x0571c45dU, 0x6f0406d4U, 0xff605015U, 0x241998fbU, 0x97d6bde9U, + 0xcc894043U, 0x7767d99eU, 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, + 0xdb79c8eeU, 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, + 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, 0xfbfd0effU, + 0x560f8538U, 0x1e3daed5U, 0x27362d39U, 0x640a0fd9U, 0x21685ca6U, + 0xd19b5b54U, 0x3a24362eU, 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, + 0x9e1b9b91U, 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, + 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, 0x0b0e090dU, + 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, 0x8557f119U, 0x4caf7507U, + 0xbbee99ddU, 0xfda37f60U, 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, + 0x345bfb7eU, 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, + 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, 0x7d854a24U, + 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, 0x4b1d9e2fU, 0xf3dcb230U, + 0xec0d8652U, 0xd077c1e3U, 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, + 0x2247e964U, 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, + 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, 0xcfa6f581U, + 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, 0xe42c3a9dU, 0x0d507892U, + 0x9b6a5fccU, 0x62547e46U, 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, + 0xf582c3afU, 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, + 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, 0x09cd2678U, + 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, 0x65e6956eU, 0x7eaaffe6U, + 0x0821bccfU, 0xe6ef15e8U, 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, + 0xd629b07cU, 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, + 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, 0x4af10498U, + 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, 0x8d764dd6U, 0x4d43efb0U, + 0x54ccaa4dU, 0xdfe49604U, 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, + 0x7f466551U, 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, + 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, 0x8c9ad761U, + 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, 0xeecea927U, 0x35b761c9U, + 0xede11ce5U, 0x3c7a47b1U, 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, + 0xbf73c737U, 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, + 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, 0x72161dc3U, + 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, 0x7139a801U, 0xde080cb3U, + 0x9cd8b4e4U, 0x906456c1U, 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, + 0x42d0b857U, +}; +static const u32 Td2[256] = { + 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, 0x6bcb3babU, + 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, 0xfa552030U, 0x6df6ad76U, + 0x769188ccU, 0x4c25f502U, 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, + 0xa38fb562U, 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, + 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, 0x5fe7038fU, + 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, 0x832dd4beU, 0x21d35874U, + 0x692949e0U, 0xc8448ec9U, 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, + 0x71dd27b9U, 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, + 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, 0x77e0b164U, + 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, 0x68587048U, 0xfd198f45U, + 0x6c8794deU, 0xf8b7527bU, 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, + 0xab2a6655U, 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, + 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, 0x1c2b8acfU, + 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, 0xf4cd65daU, 0xbed50605U, + 0x621fd134U, 0xfe8ac4a6U, 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, + 0xeb75a4f6U, 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, + + 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, 0x8db59154U, + 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, 0xfb241998U, 0xe997d6bdU, + 0x43cc8940U, 0x9e7767d9U, 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, + 0xeedb79c8U, 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, + 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, 0xfffbfd0eU, + 0x38560f85U, 0xd51e3daeU, 0x3927362dU, 0xd9640a0fU, 0xa621685cU, + 0x54d19b5bU, 0x2e3a2436U, 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, + 0x919e1b9bU, 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, + 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, 0x0d0b0e09U, + 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, 0x198557f1U, 0x074caf75U, + 0xddbbee99U, 0x60fda37fU, 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, + 0x7e345bfbU, 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, + 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, 0x247d854aU, + 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, 0x2f4b1d9eU, 0x30f3dcb2U, + 0x52ec0d86U, 0xe3d077c1U, 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, + 0x642247e9U, 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, + 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, 0x81cfa6f5U, + 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, 0x9de42c3aU, 0x920d5078U, + 0xcc9b6a5fU, 0x4662547eU, 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, + 0xaff582c3U, 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, + 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, 0x7809cd26U, + 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, 0x6e65e695U, 0xe67eaaffU, + 0xcf0821bcU, 0xe8e6ef15U, 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, + 0x7cd629b0U, 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, + 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, 0x984af104U, + 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, 0xd68d764dU, 0xb04d43efU, + 0x4d54ccaaU, 0x04dfe496U, 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, + 0x517f4665U, 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, + 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, 0x618c9ad7U, + 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, 0x27eecea9U, 0xc935b761U, + 0xe5ede11cU, 0xb13c7a47U, 0xdf599cd2U, 0x733f55f2U, 0xce791814U, + 0x37bf73c7U, 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, + 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, 0xc372161dU, + 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, 0x017139a8U, 0xb3de080cU, + 0xe49cd8b4U, 0xc1906456U, 0x84617bcbU, 0xb670d532U, 0x5c74486cU, + 0x5742d0b8U, +}; +static const u32 Td3[256] = { + 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, 0xab6bcb3bU, + 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, 0x30fa5520U, 0x766df6adU, + 0xcc769188U, 0x024c25f5U, 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, + 0x62a38fb5U, 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, + 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, 0x8f5fe703U, + 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, 0xbe832dd4U, 0x7421d358U, + 0xe0692949U, 0xc9c8448eU, 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, + 0xb971dd27U, 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, + 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, 0x6477e0b1U, + 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, 0x48685870U, 0x45fd198fU, + 0xde6c8794U, 0x7bf8b752U, 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, + 0x55ab2a66U, 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, + 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, 0xcf1c2b8aU, + 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, 0xdaf4cd65U, 0x05bed506U, + 0x34621fd1U, 0xa6fe8ac4U, 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, + 0xf6eb75a4U, 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, + 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, 0x548db591U, + 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, 0x98fb2419U, 0xbde997d6U, + 0x4043cc89U, 0xd99e7767U, 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, + 0xc8eedb79U, 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, + 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, 0x0efffbfdU, + 0x8538560fU, 0xaed51e3dU, 0x2d392736U, 0x0fd9640aU, 0x5ca62168U, + 0x5b54d19bU, 0x362e3a24U, 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, + 0x9b919e1bU, 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, + 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, 0x090d0b0eU, + 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, 0xf1198557U, 0x75074cafU, + 0x99ddbbeeU, 0x7f60fda3U, 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, + 0xfb7e345bU, 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, + 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, 0x4a247d85U, + 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, 0x9e2f4b1dU, 0xb230f3dcU, + 0x8652ec0dU, 0xc1e3d077U, 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, + 0xe9642247U, 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, + 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, 0xf581cfa6U, + 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, 0x3a9de42cU, 0x78920d50U, + 0x5fcc9b6aU, 0x7e466254U, 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, + 0xc3aff582U, 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, + 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, 0x267809cdU, + 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, 0x956e65e6U, 0xffe67eaaU, + 0xbccf0821U, 0x15e8e6efU, 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, + 0xb07cd629U, 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, + 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, 0x04984af1U, + 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, 0x4dd68d76U, 0xefb04d43U, + 0xaa4d54ccU, 0x9604dfe4U, 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, + 0x65517f46U, 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, + 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, 0xd7618c9aU, + 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, 0xa927eeceU, 0x61c935b7U, + 0x1ce5ede1U, 0x47b13c7aU, 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, + 0xc737bf73U, 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, + 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, 0x1dc37216U, + 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, 0xa8017139U, 0x0cb3de08U, + 0xb4e49cd8U, 0x56c19064U, 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, + 0xb85742d0U, +}; +static const u32 Td4[256] = { + 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, 0x30303030U, + 0x36363636U, 0xa5a5a5a5U, 0x38383838U, 0xbfbfbfbfU, 0x40404040U, + 0xa3a3a3a3U, 0x9e9e9e9eU, 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, + 0xfbfbfbfbU, 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, + 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, 0x34343434U, + 0x8e8e8e8eU, 0x43434343U, 0x44444444U, 0xc4c4c4c4U, 0xdedededeU, + 0xe9e9e9e9U, 0xcbcbcbcbU, 0x54545454U, 0x7b7b7b7bU, 0x94949494U, + 0x32323232U, 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, + 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, 0x42424242U, + 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, 0x08080808U, 0x2e2e2e2eU, + 0xa1a1a1a1U, 0x66666666U, 0x28282828U, 0xd9d9d9d9U, 0x24242424U, + 0xb2b2b2b2U, 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, + 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, 0x72727272U, + 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, 0x86868686U, 0x68686868U, + 0x98989898U, 0x16161616U, 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, + 0xccccccccU, 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, + 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, 0xfdfdfdfdU, + 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, 0x5e5e5e5eU, 0x15151515U, + 0x46464646U, 0x57575757U, 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, + 0x84848484U, 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, + 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, 0xf7f7f7f7U, + 0xe4e4e4e4U, 0x58585858U, 0x05050505U, 0xb8b8b8b8U, 0xb3b3b3b3U, + 0x45454545U, 0x06060606U, 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, + 0x8f8f8f8fU, 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, + 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, 0x01010101U, + 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, 0x3a3a3a3aU, 0x91919191U, + 0x11111111U, 0x41414141U, 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, + 0xeaeaeaeaU, 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, + 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, 0x96969696U, + 0xacacacacU, 0x74747474U, 0x22222222U, 0xe7e7e7e7U, 0xadadadadU, + 0x35353535U, 0x85858585U, 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, + 0xe8e8e8e8U, 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, + 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, 0x1d1d1d1dU, + 0x29292929U, 0xc5c5c5c5U, 0x89898989U, 0x6f6f6f6fU, 0xb7b7b7b7U, + 0x62626262U, 0x0e0e0e0eU, 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, + 0x1b1b1b1bU, 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, + 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, 0x9a9a9a9aU, + 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, 0x78787878U, 0xcdcdcdcdU, + 0x5a5a5a5aU, 0xf4f4f4f4U, 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, + 0x33333333U, 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, + 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, 0x27272727U, + 0x80808080U, 0xececececU, 0x5f5f5f5fU, 0x60606060U, 0x51515151U, + 0x7f7f7f7fU, 0xa9a9a9a9U, 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, + 0x0d0d0d0dU, 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, + 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, 0xa0a0a0a0U, + 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, 0xaeaeaeaeU, 0x2a2a2a2aU, + 0xf5f5f5f5U, 0xb0b0b0b0U, 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, + 0x3c3c3c3cU, 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, + 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, 0xbabababaU, + 0x77777777U, 0xd6d6d6d6U, 0x26262626U, 0xe1e1e1e1U, 0x69696969U, + 0x14141414U, 0x63636363U, 0x55555555U, 0x21212121U, 0x0c0c0c0cU, + 0x7d7d7d7dU, +}; +static const u32 rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000, + 0x40000000, 0x80000000, 0x1B000000, 0x36000000, /* for 128-bit blocks, + Rijndael never uses + more than 10 rcon + values */ +}; + +#define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00) + +#ifdef _MSC_VER +#define GETU32(p) SWAP(*((u32 *)(p))) +#define PUTU32(ct, st) \ + { \ + *((u32 *)(ct)) = SWAP((st)); \ + } +#else +#define GETU32(pt) \ + (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ \ + ((u32)(pt)[3])) +#define PUTU32(ct, st) \ + { \ + (ct)[0] = (u8)((st) >> 24); \ + (ct)[1] = (u8)((st) >> 16); \ + (ct)[2] = (u8)((st) >> 8); \ + (ct)[3] = (u8)(st); \ + } +#endif + +/** + * Expand the cipher key into the encryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) +{ + int i = 0; + u32 temp; + + rk[0] = GETU32(cipherKey); + rk[1] = GETU32(cipherKey + 4); + rk[2] = GETU32(cipherKey + 8); + rk[3] = GETU32(cipherKey + 12); + if (keyBits == 128) { + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + return 10; + } + rk += 4; + } + } + rk[4] = GETU32(cipherKey + 16); + rk[5] = GETU32(cipherKey + 20); + if (keyBits == 192) { + for (;;) { + temp = rk[5]; + rk[6] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[7] = rk[1] ^ rk[6]; + rk[8] = rk[2] ^ rk[7]; + rk[9] = rk[3] ^ rk[8]; + if (++i == 8) { + return 12; + } + rk[10] = rk[4] ^ rk[9]; + rk[11] = rk[5] ^ rk[10]; + rk += 6; + } + } + rk[6] = GETU32(cipherKey + 24); + rk[7] = GETU32(cipherKey + 28); + if (keyBits == 256) { + for (;;) { + temp = rk[7]; + rk[8] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp)&0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[9] = rk[1] ^ rk[8]; + rk[10] = rk[2] ^ rk[9]; + rk[11] = rk[3] ^ rk[10]; + if (++i == 7) { + return 14; + } + temp = rk[11]; + rk[12] = rk[4] ^ (Te4[(temp >> 24)] & 0xff000000) ^ + (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(temp)&0xff] & 0x000000ff); + rk[13] = rk[5] ^ rk[12]; + rk[14] = rk[6] ^ rk[13]; + rk[15] = rk[7] ^ rk[14]; + + rk += 8; + } + } + return 0; +} + +/** + * Expand the cipher key into the decryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) +{ + int Nr, i, j; + u32 temp; + + /* expand the cipher key: */ + Nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits); + /* invert the order of the round keys: */ + for (i = 0, j = 4 * Nr; i < j; i += 4, j -= 4) { + temp = rk[i]; + rk[i] = rk[j]; + rk[j] = temp; + temp = rk[i + 1]; + rk[i + 1] = rk[j + 1]; + rk[j + 1] = temp; + temp = rk[i + 2]; + rk[i + 2] = rk[j + 2]; + rk[j + 2] = temp; + temp = rk[i + 3]; + rk[i + 3] = rk[j + 3]; + rk[j + 3] = temp; + } + /* apply the inverse MixColumn transform to all round keys but the first + * and the last: */ + for (i = 1; i < Nr; i++) { + rk += 4; + rk[0] = Td0[Te4[(rk[0] >> 24)] & 0xff] ^ + Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[0]) & 0xff] & 0xff]; + rk[1] = Td0[Te4[(rk[1] >> 24)] & 0xff] ^ + Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[1]) & 0xff] & 0xff]; + rk[2] = Td0[Te4[(rk[2] >> 24)] & 0xff] ^ + Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[2]) & 0xff] & 0xff]; + rk[3] = Td0[Te4[(rk[3] >> 24)] & 0xff] ^ + Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[3]) & 0xff] & 0xff]; + } + return Nr; +} + +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]) +{ + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(pt) ^ rk[0]; + s1 = GETU32(pt + 4) ^ rk[1]; + s2 = GETU32(pt + 8) ^ rk[2]; + s3 = GETU32(pt + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[4]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[5]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[6]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[8]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[9]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[10]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[12]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[13]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[14]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[16]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[17]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[18]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[20]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[21]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[22]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[24]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[25]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[26]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[28]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[29]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[30]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[32]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[33]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[34]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[36]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[37]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[38]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[40]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[41]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[42]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[44]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[45]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[46]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[(s3)&0xff] ^ rk[4]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[(s0)&0xff] ^ rk[5]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[(s1)&0xff] ^ rk[6]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[(s2)&0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Te0[(t0 >> 24)] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[(t3)&0xff] ^ rk[0]; + s1 = Te0[(t1 >> 24)] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[(t0)&0xff] ^ rk[1]; + s2 = Te0[(t2 >> 24)] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[(t1)&0xff] ^ rk[2]; + s3 = Te0[(t3 >> 24)] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[(t2)&0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Te4[(t0 >> 24)] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t3)&0xff] & 0x000000ff) ^ rk[0]; + PUTU32(ct, s0); + s1 = (Te4[(t1 >> 24)] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t0)&0xff] & 0x000000ff) ^ rk[1]; + PUTU32(ct + 4, s1); + s2 = (Te4[(t2 >> 24)] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t1)&0xff] & 0x000000ff) ^ rk[2]; + PUTU32(ct + 8, s2); + s3 = (Te4[(t3 >> 24)] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t2)&0xff] & 0x000000ff) ^ rk[3]; + PUTU32(ct + 12, s3); +} + +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]) +{ + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(ct) ^ rk[0]; + s1 = GETU32(ct + 4) ^ rk[1]; + s2 = GETU32(ct + 8) ^ rk[2]; + s3 = GETU32(ct + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[4]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[5]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[6]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[8]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[9]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[10]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[12]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[13]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[14]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[16]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[17]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[18]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[20]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[21]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[22]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[24]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[25]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[26]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[28]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[29]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[30]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[32]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[33]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[34]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[36]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[37]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[38]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[40]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[41]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[42]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[44]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[45]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[46]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[48]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[49]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[50]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[52]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[53]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[54]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[(s1)&0xff] ^ rk[4]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[(s2)&0xff] ^ rk[5]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[(s3)&0xff] ^ rk[6]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[(s0)&0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Td0[(t0 >> 24)] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[(t1)&0xff] ^ rk[0]; + s1 = Td0[(t1 >> 24)] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[(t2)&0xff] ^ rk[1]; + s2 = Td0[(t2 >> 24)] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[(t3)&0xff] ^ rk[2]; + s3 = Td0[(t3 >> 24)] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[(t0)&0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Td4[(t0 >> 24)] & 0xff000000) ^ + (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t1)&0xff] & 0x000000ff) ^ rk[0]; + PUTU32(pt, s0); + s1 = (Td4[(t1 >> 24)] & 0xff000000) ^ + (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t2)&0xff] & 0x000000ff) ^ rk[1]; + PUTU32(pt + 4, s1); + s2 = (Td4[(t2 >> 24)] & 0xff000000) ^ + (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t3)&0xff] & 0x000000ff) ^ rk[2]; + PUTU32(pt + 8, s2); + s3 = (Td4[(t3 >> 24)] & 0xff000000) ^ + (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(t0)&0xff] & 0x000000ff) ^ rk[3]; + PUTU32(pt + 12, s3); +} + +#ifdef INTERMEDIATE_VALUE_KAT + +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) +{ + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr - 1); r > 0; r--) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[(s3)&0xff] ^ rk[0]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[(s0)&0xff] ^ rk[1]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[(s1)&0xff] ^ rk[2]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[(s2)&0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * apply last round and + * map cipher state to byte array block: + */ + if (rounds == Nr) { + t0 = (Te4[(s0 >> 24)] & 0xff000000) ^ + (Te4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s3)&0xff] & 0x000000ff) ^ rk[0]; + t1 = (Te4[(s1 >> 24)] & 0xff000000) ^ + (Te4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s0)&0xff] & 0x000000ff) ^ rk[1]; + t2 = (Te4[(s2 >> 24)] & 0xff000000) ^ + (Te4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s1)&0xff] & 0x000000ff) ^ rk[2]; + t3 = (Te4[(s3 >> 24)] & 0xff000000) ^ + (Te4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s2)&0xff] & 0x000000ff) ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + } + + PUTU32(block, s0); + PUTU32(block + 4, s1); + PUTU32(block + 8, s2); + PUTU32(block + 12, s3); +} + +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) +{ + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr) - 1; r > 0; r--) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[(s1)&0xff] ^ rk[0]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[(s2)&0xff] ^ rk[1]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[(s3)&0xff] ^ rk[2]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[(s0)&0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * complete the last round and + * map cipher state to byte array block: + */ + t0 = (Td4[(s0 >> 24)] & 0xff000000) ^ + (Td4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s1)&0xff] & 0x000000ff); + t1 = (Td4[(s1 >> 24)] & 0xff000000) ^ + (Td4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s2)&0xff] & 0x000000ff); + t2 = (Td4[(s2 >> 24)] & 0xff000000) ^ + (Td4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s3)&0xff] & 0x000000ff); + t3 = (Td4[(s3 >> 24)] & 0xff000000) ^ + (Td4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[(s0)&0xff] & 0x000000ff); + + if (rounds == Nr) { + t0 ^= rk[0]; + t1 ^= rk[1]; + t2 ^= rk[2]; + t3 ^= rk[3]; + } + + PUTU32(block, t0); + PUTU32(block + 4, t1); + PUTU32(block + 8, t2); + PUTU32(block + 12, t3); +} + +#endif /* INTERMEDIATE_VALUE_KAT */ diff --git a/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.h b/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.h new file mode 100644 index 0000000..eabff49 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/rijndael-alg-fst.h @@ -0,0 +1,53 @@ +/** + * rijndael-alg-fst.h + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen <[email protected]> + * @author Antoon Bosselaers <[email protected]> + * @author Paulo Barreto <[email protected]> + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef RIJNDAEL_ALG_FST_H +#define RIJNDAEL_ALG_FST_H + +#define MAXKC (256 / 32) +#define MAXKB (256 / 8) +#define MAXNR 14 + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; + +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]); +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]); + +#ifdef INTERMEDIATE_VALUE_KAT +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +#endif /* INTERMEDIATE_VALUE_KAT */ + +#endif /* RIJNDAEL_ALG_FST_H */ diff --git a/rdns_scan/zmap4rdns/lib/types.h b/rdns_scan/zmap4rdns/lib/types.h new file mode 100644 index 0000000..ef4ec2c --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/types.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_TYPES_H +#define ZMAP_TYPES_H + +#include <stdint.h> + +#include <stdint.h> + +typedef uint32_t ipaddr_n_t; // IPv4 address network order +typedef uint32_t ipaddr_h_t; // IPv4 address host order +typedef uint16_t port_n_t; // port network order +typedef uint16_t port_h_t; // port host order +typedef unsigned char macaddr_t; + +#endif /* ZMAP_TYPES_H */ diff --git a/rdns_scan/zmap4rdns/lib/util.c b/rdns_scan/zmap4rdns/lib/util.c new file mode 100644 index 0000000..0a1403f --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/util.c @@ -0,0 +1,343 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include "util.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "includes.h" +#include "xalloc.h" + +#include <errno.h> +#include <unistd.h> +#include <sched.h> +#include <pthread.h> +#include <sys/types.h> +#include <pwd.h> + +#include "../lib/logger.h" + +#define MAX_SPLITS 128 + +int max_int(int a, int b) +{ + if (a >= b) { + return a; + } + return b; +} + +void enforce_range(const char *name, int v, int min, int max) +{ + if (check_range(v, min, max) == EXIT_FAILURE) { + log_fatal("zmap", "argument `%s' must be between %d and %d\n", + name, min, max); + } +} + +void split_string(const char *in, int *len, const char ***results) +{ + const char **fields = xcalloc(MAX_SPLITS, sizeof(const char *)); + int retvlen = 0; + const char *currloc = in; + // parse csv into a set of strings + while (1) { + assert(retvlen < MAX_SPLITS); + size_t len = strcspn(currloc, ", "); + if (len == 0) { + currloc++; + } else { + char *new = xmalloc(len + 1); + strncpy(new, currloc, len); + new[len] = '\0'; + fields[retvlen++] = new; + assert(fields[retvlen - 1]); + } + if (len == strlen(currloc)) { + break; + } + currloc += len; + } + *results = fields; + *len = retvlen; +} + +void fprintw(FILE *f, const char *s, size_t w) +{ + if (strlen(s) <= w) { + fprintf(f, "%s", s); + return; + } + // process each line individually in order to + // respect existing line breaks in string. + char *news = strdup(s); + char *pch = strtok(news, "\n"); + while (pch) { + if (strlen(pch) <= w) { + printf("%s\n", pch); + pch = strtok(NULL, "\n"); + continue; + } + char *t = pch; + while (strlen(t)) { + size_t numchars = 0; // number of chars to print + char *tmp = t; + while (1) { + size_t new = strcspn(tmp, " ") + 1; + if (new == strlen(tmp) || new > w) { + // there are no spaces in the string, + // so, just print the entire thing on + // one line; + numchars += new; + break; + } else if (numchars + new > w) { + // if we added any more, we'd be over w + // chars so time to print the line and + // move on to the next. + break; + } else { + tmp += (size_t) new; + numchars += new; + } + } + fprintf(f, "%.*s\n", (int)numchars, t); + t += (size_t)numchars; + if (t > pch + (size_t)strlen(pch)) { + break; + } + } + pch = strtok(NULL, "\n"); + } + free(news); +} + +uint32_t parse_max_hosts(char *max_targets) +{ + char *end; + errno = 0; + double v = strtod(max_targets, &end); + if (end == max_targets || errno != 0) { + log_fatal("argparse", "can't convert max-targets to a number"); + } + if (end[0] == '%' && end[1] == '\0') { + // treat as percentage + v = v * ((unsigned long long int)1 << 32) / 100.; + } else if (end[0] != '\0') { + log_fatal("eargparse", "extra characters after max-targets"); + } + if (v <= 0) { + return 0; + } else if (v >= ((unsigned long long int)1 << 32)) { + return 0xFFFFFFFF; + } else { + return v; + } +} + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len) +{ + int y = time / 31556736; + int d = (time % 31556736) / 86400; + int h = (time % 86400) / 3600; + int m = (time % 3600) / 60; + int s = time % 60; + + if (est) { + if (y > 0) { + snprintf(buf, len, "%d years", y); + } else if (d > 9) { + snprintf(buf, len, "%dd", d); + } else if (d > 0) { + snprintf(buf, len, "%dd%02dh", d, h); + } else if (h > 9) { + snprintf(buf, len, "%dh", h); + } else if (h > 0) { + snprintf(buf, len, "%dh%02dm", h, m); + } else if (m > 9) { + snprintf(buf, len, "%dm", m); + } else if (m > 0) { + snprintf(buf, len, "%dm%02ds", m, s); + } else { + snprintf(buf, len, "%ds", s); + } + } else { + if (d > 0) { + snprintf(buf, len, "%dd%d:%02d:%02d", d, h, m, s); + } else if (h > 0) { + snprintf(buf, len, "%d:%02d:%02d", h, m, s); + } else { + snprintf(buf, len, "%d:%02d", m, s); + } + } +} + +// pretty print quantities +void number_string(uint32_t n, char *buf, size_t len) +{ + int figs = 0; + if (n < 1000) { + snprintf(buf, len, "%u ", n); + } else if (n < 1000000) { + if (n < 10000) { + figs = 2; + } else if (n < 100000) { + figs = 1; + } + snprintf(buf, len, "%0.*f K", figs, (float)n / 1000.); + } else { + if (figs < 10000000) { + figs = 2; + } else if (figs < 100000000) { + figs = 1; + } + snprintf(buf, len, "%0.*f M", figs, (float)n / 1000000.); + } +} + +int parse_mac(macaddr_t *out, char *in) +{ + if (strlen(in) < MAC_ADDR_LEN * 3 - 1) + return 0; + char octet[4]; + octet[2] = '\0'; + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i < MAC_ADDR_LEN - 1 && in[i * 3 + 2] != ':') { + return 0; + } + strncpy(octet, &in[i * 3], 2); + char *err = NULL; + long b = strtol(octet, &err, 16); + if (err && *err != '\0') { + return 0; + } + out[i] = b & 0xFF; + } + return 1; +} + +int check_range(int v, int min, int max) +{ + if (v < min || v > max) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int file_exists(char *name) +{ + FILE *file = fopen(name, "r"); + if (!file) + return 0; + fclose(file); + return 1; +} + +#if defined(__APPLE__) +#include <uuid/uuid.h> +#endif + +int drop_privs() +{ + struct passwd *pw; + if (geteuid() != 0) { + /* Not root */ + return EXIT_SUCCESS; + } + if ((pw = getpwnam("nobody")) != NULL) { + if (setuid(pw->pw_uid) == 0) { + return EXIT_SUCCESS; // success + } + } + return EXIT_FAILURE; +} + +#if defined(__APPLE__) + +#include <mach/thread_act.h> + +int set_cpu(uint32_t core) +{ + mach_port_t tid = pthread_mach_thread_np(pthread_self()); + struct thread_affinity_policy policy; + policy.affinity_tag = core; + kern_return_t ret = thread_policy_set(tid, THREAD_AFFINITY_POLICY, + (thread_policy_t)&policy, + THREAD_AFFINITY_POLICY_COUNT); + if (ret != KERN_SUCCESS) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#elif defined(__DragonFly__) + +#include <sys/usched.h> + +int set_cpu(uint32_t core) +{ + if (usched_set(getpid(), USCHED_SET_CPU, &core, sizeof(core)) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#elif defined(__NetBSD__) + +int set_cpu(uint32_t core) +{ + cpuset_t *cpuset = cpuset_create(); + if (cpuset == NULL) { + return EXIT_FAILURE; + } + cpuset_zero(cpuset); + cpuset_set(core, cpuset); + cpuset_destroy(cpuset); + + if (pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), + cpuset) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#else + +#if defined(__FreeBSD__) +#include <sys/param.h> +#include <sys/cpuset.h> +#include <pthread_np.h> +#define cpu_set_t cpuset_t +#endif + +int set_cpu(uint32_t core) +{ + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(core, &cpuset); + + if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), + &cpuset) != 0) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +#endif diff --git a/rdns_scan/zmap4rdns/lib/util.h b/rdns_scan/zmap4rdns/lib/util.h new file mode 100644 index 0000000..f703ba1 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/util.h @@ -0,0 +1,59 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_UTIL_H +#define ZMAP_UTIL_H + +#include <stdio.h> +#include <stdint.h> + +#include "types.h" + +int max_int(int a, int b); + +uint32_t parse_max_hosts(char *max_targets); +void enforce_range(const char *name, int v, int min, int max); + +// Splits comma delimited string into char*[]. Does not handle +// escaping or complicated setups - designed to process a set +// of fields that the user wants output +void split_string(const char *in, int *len, const char ***results); + +// Print a string using w length long lines, attempting to break on +// spaces +void fprintw(FILE *f, const char *s, size_t w); + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len); + +// pretty print quantities +void number_string(uint32_t n, char *buf, size_t len); + +// Convert a string representation of a MAC address to a byte array +int parse_mac(macaddr_t *out, char *in); + +int check_range(int v, int min, int max); + +int file_exists(char *name); + +// If running as root, drops privileges to that of user "nobody". +// Otherwise, does nothing. +int drop_privs(); + +// Set CPU affinity to a single core +int set_cpu(uint32_t core); + +#endif /* ZMAP_UTIL_H */ diff --git a/rdns_scan/zmap4rdns/lib/xalloc.c b/rdns_scan/zmap4rdns/lib/xalloc.c new file mode 100644 index 0000000..6542e74 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/xalloc.c @@ -0,0 +1,55 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "xalloc.h" +#include "logger.h" + +#include <stdlib.h> +#include <string.h> + +static void die() __attribute__((noreturn)); + +void *xcalloc(size_t count, size_t size) +{ + void *res = calloc(count, size); + if (res == NULL) { + die(); + } + return res; +} + +void xfree(void *ptr) { free(ptr); } + +void *xmalloc(size_t size) +{ + void *res = malloc(size); + if (res == NULL) { + die(); + } + memset(res, 0, size); + return res; +} + +void *xrealloc(void *ptr, size_t size) +{ + void *res = realloc(ptr, size); + if (res == NULL) { + die(); + } + return res; +} + +void die() { log_fatal("zmap", "Out of memory"); } diff --git a/rdns_scan/zmap4rdns/lib/xalloc.h b/rdns_scan/zmap4rdns/lib/xalloc.h new file mode 100644 index 0000000..5dd6001 --- /dev/null +++ b/rdns_scan/zmap4rdns/lib/xalloc.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_ALLOC_H +#define ZMAP_ALLOC_H + +#include <stddef.h> + +void *xcalloc(size_t count, size_t size); + +void xfree(void *ptr); + +void *xmalloc(size_t size); + +void *xrealloc(void *ptr, size_t size); + +#endif /* ZMAP_ALLOC_H */ diff --git a/rdns_scan/zmap4rdns/scripts/check_manfile.py b/rdns_scan/zmap4rdns/scripts/check_manfile.py new file mode 100644 index 0000000..205b7da --- /dev/null +++ b/rdns_scan/zmap4rdns/scripts/check_manfile.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# CI runs this script to verify that options appearing in ZTools' ggo.in files +# also appear in their .ronn files. It does not check that `make manpages` has +# actually been run. +# +# This script assumes it's being run from the root of the zmap repository. +# + +import sys + +checks = [ + ("zopt.ggo.in", "zmap.1.ronn"), + ("zbopt.ggo.in", "zblocklist.1.ronn"), + ("zitopt.ggo.in", "ziterate.1.ronn"), + ("ztopt.ggo.in", "ztee.1.ronn") +] + +failures = False + +for ggo, ronn in checks: + options = [] + with open("src/" + ggo) as fd: + for l in fd: + if l.startswith("option "): + option = l.split()[1].lstrip('"').rstrip('"') + options.append(option) + + man = open("src/" + ronn).read() + + for option in options: + if option not in man: + failures = True + sys.stderr.write(f"option \"{option}\" is present in \"{ggo}\" but missing from man file \"{ronn}\"\n") + sys.stderr.flush() + +if failures: + sys.exit(1) diff --git a/rdns_scan/zmap4rdns/src/.gitignore b/rdns_scan/zmap4rdns/src/.gitignore new file mode 100644 index 0000000..87de78c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/.gitignore @@ -0,0 +1,4 @@ +lexer.c +lexer.h +parser.c +parser.h diff --git a/rdns_scan/zmap4rdns/src/CMakeLists.txt b/rdns_scan/zmap4rdns/src/CMakeLists.txt new file mode 100644 index 0000000..2c8eac2 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/CMakeLists.txt @@ -0,0 +1,273 @@ +include_directories( + "${CMAKE_CURRENT_BINARY_DIR}" + ${PROJECT_SOURCE_DIR}/lib + ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/src/output_modules + ${PROJECT_SOURCE_DIR}/src/tests +) + +# ADD YOUR PROBE MODULE HERE +set(EXTRA_PROBE_MODULES +) + +# ADD YOUR OUTPUT MODULE HERE +set(EXTRA_OUTPUT_MODULES + output_module/module_sql.c +) + +set(OUTPUT_MODULE_SOURCES + output_modules/module_csv.c + output_modules/module_json.c + output_modules/output_modules.c +) + +set(PROBE_MODULE_SOURCES + probe_modules/module_icmp_echo.c + probe_modules/module_icmp_echo_time.c + probe_modules/module_tcp_synscan.c + probe_modules/module_tcp_synackscan.c + #probe_modules/module_tcp_cisco_backdoor.c + probe_modules/module_udp.c + probe_modules/packet.c + probe_modules/probe_modules.c + probe_modules/module_ntp.c + probe_modules/module_upnp.c + probe_modules/module_dns.c + probe_modules/module_bacnet.c +) + +set(SOURCES + aesrand.c + cyclic.c + expression.c + fieldset.c + filter.c + get_gateway.c + iterator.c + monitor.c + recv.c + send.c + shard.c + socket.c + state.c + summary.c + utility.c + validate.c + zmap.c + zopt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/zopt.h" + "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" + "${CMAKE_CURRENT_BINARY_DIR}/parser.c" + ${EXTRA_PROBE_MODULES} + ${EXTRA_OUTPUT_MODULES} + ${PROBE_MODULE_SOURCES} + ${OUTPUT_MODULE_SOURCES} +) + +set(ZTESTSOURCES + aesrand.c + cyclic.c + expression.c + fieldset.c + filter.c + get_gateway.c + iterator.c + monitor.c + recv.c + send.c + shard.c + socket.c + state.c + summary.c + validate.c + ztopt_compat.c + ${PROBE_MODULE_SOURCES} + ${OUTPUT_MODULE_SOURCES} + tests/test_harness.c + "${CMAKE_CURRENT_BINARY_DIR}/ztopt.h" + "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" + "${CMAKE_CURRENT_BINARY_DIR}/parser.c" + ${EXTRA_PROBE_MODULES} + ${EXTRA_OUTPUT_MODULES} +) + +set(ZBLSOURCES + zblocklist.c + zbopt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/zbopt.h" +) + +set(ZITSOURCES + aesrand.c + cyclic.c + iterator.c + shard.c + state.c + validate.c + zitopt_compat.c + ziterate.c + "${CMAKE_CURRENT_BINARY_DIR}/zitopt.h" +) + +set(ZTEESOURCES + ztee.c + topt_compat.c + "${CMAKE_CURRENT_BINARY_DIR}/topt.h" +) + +# Handle various versions of socket +if(WITH_PFRING) + set(SOURCES ${SOURCES} socket-pfring.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-pfring.c) +elseif (APPLE OR BSD) + set(SOURCES ${SOURCES} socket-bsd.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-bsd.c) +else() + set(SOURCES ${SOURCES} socket-linux.c) + set(ZTESTSOURCES ${ZTESTSOURCES} socket-linux.c) +endif() + +# Handle various versions of recv +if(WITH_PFRING) + set(SOURCES ${SOURCES} recv-pfring.c) + set(ZTESTSOURCES ${ZTESTSOURCES} recv-pfring.c) +else() + set(SOURCES ${SOURCES} recv-pcap.c) + set(ZTESTSOURCES ${ZTESTSOURCES} recv-pcap.c) +endif() + +# Set configure time zmap version +configure_file(topt.ggo.in ${CMAKE_BINARY_DIR}/src/topt.ggo @ONLY) +configure_file(zbopt.ggo.in ${CMAKE_BINARY_DIR}/src/zbopt.ggo @ONLY) +configure_file(zitopt.ggo.in ${CMAKE_BINARY_DIR}/src/zitopt.ggo @ONLY) +configure_file(zopt.ggo.in ${CMAKE_BINARY_DIR}/src/zopt.ggo @ONLY) +configure_file(ztopt.ggo.in ${CMAKE_BINARY_DIR}/src/ztopt.ggo @ONLY) +# Additional ggo.in's should be added here and CMakeVersion.txt + +# This sets a *build* time dependency that checks git +if("${ZMAP_VERSION}" STREQUAL "DEVELOPMENT") + add_custom_target(git_versioning ALL + COMMAND ${CMAKE_COMMAND} -D ORIG_SRC_DIR:STRING="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/src/CMakeVersion.txt" + ) +endif() + +add_custom_command(OUTPUT zopt.h + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_BINARY_DIR}/zopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zopt.ggo" +) + +add_custom_command(OUTPUT topt.h + COMMAND gengetopt -S --no-help --no-version --unamed-opts=FILE -i "${CMAKE_CURRENT_BINARY_DIR}/topt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/topt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/topt.ggo" +) + +add_custom_command(OUTPUT zbopt.h + COMMAND gengetopt -C --no-help --no-version -i "${CMAKE_CURRENT_BINARY_DIR}/zbopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zbopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zbopt.ggo" +) + +add_custom_command(OUTPUT zitopt.h + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_BINARY_DIR}/zitopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/zitopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/zitopt.ggo" +) + +add_custom_command(OUTPUT ztopt.h + COMMAND gengetopt -C --no-help --no-version -i "${CMAKE_CURRENT_BINARY_DIR}/ztopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/ztopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ztopt.ggo" +) + +add_custom_command(OUTPUT lexer.c + COMMAND flex -o "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" --header-file="${CMAKE_CURRENT_BINARY_DIR}/lexer.h" "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" +) + +if(NetBSD) +add_custom_command(OUTPUT parser.c + COMMAND yacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" +) +else() +add_custom_command(OUTPUT parser.c + COMMAND byacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" +) +endif() + +add_custom_target(manpages ronn "${CMAKE_CURRENT_SOURCE_DIR}/zmap.1.ronn" --organization="ZMap" --manual="zmap" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/zblocklist.1.ronn" --organization="ZMap" --manual="zblocklist" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/ziterate.1.ronn" --organization="ZMap" --manual="ziterate" + COMMAND ronn "${CMAKE_CURRENT_SOURCE_DIR}/ztee.1.ronn" --organization="ZMap" --manual="ztee" + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/zblocklist.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/ziterate.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/zmap.1.ronn" "${CMAKE_CURRENT_SOURCE_DIR}/ztee.1.ronn" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" +) + +add_executable(zmap ${SOURCES}) +add_executable(zblocklist ${ZBLSOURCES}) +add_executable(ziterate ${ZITSOURCES}) +add_executable(ztee ${ZTEESOURCES}) +add_executable(ztests ${ZTESTSOURCES}) + +if(APPLE OR BSD) + set(DNET_LIBRARIES "dnet") +else() + set(DNET_LIBRARIES "") +endif() + +target_link_libraries( + zmap + zmaplib + ${PFRING_LIBRARIES} + pcap gmp m unistring + mysqlclient + ${DNET_LIBRARIES} + ${JSON_LIBRARIES} +) + +target_link_libraries( + zblocklist + zmaplib + m +) + +target_link_libraries( + ziterate + zmaplib + gmp + m +) + +target_link_libraries( + ztee + zmaplib + m +) + +target_link_libraries( + ztests + zmaplib + ${PFRING_LIBRARIES} + pcap gmp m unistring + mysqlclient + ${DNET_LIBRARIES} + ${JSON_LIBRARIES} +) + +# Install binary +install( + TARGETS + zmap + zblocklist + ziterate + ztee + RUNTIME DESTINATION sbin +) + +# Install Manpages +install( + FILES + zmap.1 + zblocklist.1 + ziterate.1 + ztee.1 + DESTINATION share/man/man1 +) diff --git a/rdns_scan/zmap4rdns/src/CMakeVersion.txt b/rdns_scan/zmap4rdns/src/CMakeVersion.txt new file mode 100644 index 0000000..893bc2f --- /dev/null +++ b/rdns_scan/zmap4rdns/src/CMakeVersion.txt @@ -0,0 +1,15 @@ +set(GIT_CMD "git") +set(GIT_ARGS "log" "-n" "1" "--pretty=format:%h - %ad") +execute_process(COMMAND ${GIT_CMD} ${GIT_ARGS} + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_COMMIT) + if (GIT_RESULT) + set (GIT_COMMIT "UNKNOWN") + endif() +set(ZMAP_VERSION "Development Build. Commit ${GIT_COMMIT}") + +configure_file("${ORIG_SRC_DIR}/src/topt.ggo.in" "${CMAKE_BINARY_DIR}/topt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zbopt.ggo.in" "${CMAKE_BINARY_DIR}/zbopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zitopt.ggo.in" "${CMAKE_BINARY_DIR}/zitopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/zopt.ggo.in" "${CMAKE_BINARY_DIR}/zopt.ggo" @ONLY) +configure_file("${ORIG_SRC_DIR}/src/ztopt.ggo.in" "${CMAKE_BINARY_DIR}/ztopt.ggo" @ONLY) diff --git a/rdns_scan/zmap4rdns/src/aesrand.c b/rdns_scan/zmap4rdns/src/aesrand.c new file mode 100644 index 0000000..2552db4 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/aesrand.c @@ -0,0 +1,71 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> +#include <string.h> + +#include "../lib/logger.h" +#include "../lib/rijndael-alg-fst.h" +#include "../lib/random.h" +#include "../lib/xalloc.h" + +#include "aesrand.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 +#define AES_KEY_BITS (AES_KEY_BYTES * 8) +#define OUTPUT_BYTES 16 + +struct aesrand { + uint32_t input[AES_BLOCK_WORDS]; + uint32_t sched[(AES_ROUNDS + 1) * 4]; + uint8_t output[OUTPUT_BYTES]; +}; + +static aesrand_t *_aesrand_init(uint8_t *key) +{ + aesrand_t *aes = xmalloc(sizeof(aesrand_t)); + memset(&aes->input, 0, AES_BLOCK_WORDS * 4); + if (rijndaelKeySetupEnc(aes->sched, key, AES_KEY_BITS) != AES_ROUNDS) { + log_fatal("aesrand", "could not initialize AES key"); + } + memset(aes->output, 0, OUTPUT_BYTES); + return aes; +} + +aesrand_t *aesrand_init_from_seed(uint64_t seed) +{ + uint8_t key[AES_KEY_BYTES]; + memset(key, 0, AES_KEY_BYTES); + for (uint8_t i = 0; i < sizeof(seed); ++i) { + key[i] = (uint8_t)((seed >> 8 * i) & 0xFF); + } + return _aesrand_init(key); +} + +aesrand_t *aesrand_init_from_random() +{ + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("aesrand", "Couldn't get random bytes"); + } + return _aesrand_init(key); +} + +uint64_t aesrand_getword(aesrand_t *aes) +{ + memcpy(aes->input, aes->output, sizeof(aes->input)); + rijndaelEncrypt(aes->sched, AES_ROUNDS, (uint8_t *)aes->input, + aes->output); + uint64_t retval; + memcpy(&retval, aes->output, sizeof(retval)); + return retval; +} diff --git a/rdns_scan/zmap4rdns/src/aesrand.h b/rdns_scan/zmap4rdns/src/aesrand.h new file mode 100644 index 0000000..5f33db3 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/aesrand.h @@ -0,0 +1,24 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdint.h> + +#ifndef AESRAND_H +#define AESRAND_H + +typedef struct aesrand aesrand_t; + +aesrand_t *aesrand_init_from_random(); + +aesrand_t *aesrand_init_from_seed(uint64_t); + +uint64_t aesrand_getword(aesrand_t *aes); + +aesrand_t *aesrand_free(aesrand_t *aes); + +#endif diff --git a/rdns_scan/zmap4rdns/src/cyclic.c b/rdns_scan/zmap4rdns/src/cyclic.c new file mode 100644 index 0000000..f6698d0 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/cyclic.c @@ -0,0 +1,172 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * cyclic provides an inexpensive approach to iterating over the IPv4 address + * space in a random(-ish) manner such that we connect to every host once in + * a scan execution without having to keep track of the IPs that have been + * scanned or need to be scanned and such that each scan has a different + * ordering. We accomplish this by utilizing a cyclic multiplicative group + * of integers modulo a prime and generating a new primitive root (generator) + * for each scan. + * + * We know that 3 is a generator of (Z mod 2^32 + 15 - {0}, *) + * and that we have coverage over the entire address space because 2**32 + 15 + * is prime and ||(Z mod PRIME - {0}, *)|| == PRIME - 1. Therefore, we + * just need to find a new generator (primitive root) of the cyclic group for + * each scan that we perform. + * + * Because generators map to generators over an isomorphism, we can efficiently + * find random primitive roots of our mult. group by finding random generators + * of the group (Zp-1, +) which is isomorphic to (Zp*, *). Specifically the + * generators of (Zp-1, +) are { s | (s, p-1) == 1 } which implies that + * the generators of (Zp*, *) are { d^s | (s, p-1) == 1 }. where d is a known + * generator of the multiplicative group. We efficiently find + * generators of the additive group by precalculating the psub1_f of + * p - 1 and randomly checking random numbers against the psub1_f until + * we find one that is coprime and map it into Zp*. Because + * totient(totient(p)) ~= 10^9, this should take relatively few + * iterations to find a new generator. + */ + +#include "cyclic.h" + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <time.h> +#include <assert.h> +#include <string.h> +#include <math.h> + +#include <gmp.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" + +// We will pick the first cyclic group from this list that is +// larger than the number of IPs in our allowlist. E.g. for an +// entire Internet scan, this would be cyclic32 +// Note: this list should remain ordered by size (primes) ascending. + +static cyclic_group_t groups[] = {{// 2^8 + 1 + .prime = 257, + .known_primroot = 3, + .prime_factors = {2}, + .num_prime_factors = 1}, + {// 2^16 + 1 + .prime = 65537, + .known_primroot = 3, + .prime_factors = {2}, + .num_prime_factors = 1}, + {// 2^24 + 43 + .prime = 16777259, + .known_primroot = 2, + .prime_factors = {2, 23, 103, 3541}, + .num_prime_factors = 4}, + {// 2^28 + 3 + .prime = 268435459, + .known_primroot = 2, + .prime_factors = {2, 3, 19, 87211}, + .num_prime_factors = 4}, + {// 2^32 + 15 + .prime = 4294967311, + .known_primroot = 3, + .prime_factors = {2, 3, 5, 131, 364289}, + .num_prime_factors = 5}}; + +#define COPRIME 1 +#define NOT_COPRIME 0 + +// Check whether an integer is coprime with (p - 1) +static int check_coprime(uint64_t check, const cyclic_group_t *group) +{ + if (check == 0 || check == 1) { + return NOT_COPRIME; + } + for (unsigned i = 0; i < group->num_prime_factors; i++) { + if (group->prime_factors[i] > check && + !(group->prime_factors[i] % check)) { + return NOT_COPRIME; + } else if (group->prime_factors[i] < check && + !(check % group->prime_factors[i])) { + return NOT_COPRIME; + } else if (group->prime_factors[i] == check) { + return NOT_COPRIME; + } + } + return COPRIME; +} + +// Return a (random) number coprime with (p - 1) of the group, +// which is a generator of the additive group mod (p - 1) +static uint32_t find_primroot(const cyclic_group_t *group, aesrand_t *aes) +{ + uint32_t candidate = + (uint32_t)((aesrand_getword(aes) & 0xFFFFFFFF) % group->prime); + uint64_t retv = 0; + + // The maximum primitive root we can return needs to be small enough such + // that there is no overflow when multiplied by any element in the largest + // group in ZMap, which currently has p = 2^{32} + 15. + const uint64_t max_root = (UINT64_C(1) << 32) - 14; + + // Repeatedly find a generator until we hit one that is small enough. For + // the largest group, we have a very low probability of ever executing this + // loop more than once, and for small groups it will only execute once. + do { + // Find an element that is coprime in the additive group + while (check_coprime(candidate, group) != COPRIME) { + candidate += 1; + candidate %= group->prime; + } + // Given a coprime element, apply the isomorphism. + retv = isomorphism(candidate, group); + } while (retv > max_root); + return retv; +} + +const cyclic_group_t *get_group(uint64_t min_size) +{ + for (unsigned i = 0; i < sizeof(groups); ++i) { + if (groups[i].prime > min_size) { + return &groups[i]; + } + } + // Should not reach, final group should always be larger than 2^32 + assert(0); +} + +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes) +{ + cycle_t cycle; + cycle.group = group; + cycle.generator = find_primroot(group, aes); + cycle.offset = (uint32_t)(aesrand_getword(aes) & 0xFFFFFFFF); + cycle.offset %= group->prime; + cycle.order = group->prime - 1; + return cycle; +} + +uint64_t isomorphism(uint64_t additive_elt, const cyclic_group_t *mult_group) +{ + assert(additive_elt < mult_group->prime); + mpz_t base, power, prime, primroot; + mpz_init_set_ui(base, mult_group->known_primroot); + mpz_init_set_ui(power, additive_elt); + mpz_init_set_ui(prime, mult_group->prime); + mpz_init(primroot); + mpz_powm(primroot, base, power, prime); + uint64_t retv = (uint64_t)mpz_get_ui(primroot); + log_debug("zmap", "Isomorphism: %llu", retv); + mpz_clear(base); + mpz_clear(power); + mpz_clear(prime); + mpz_clear(primroot); + return retv; +} diff --git a/rdns_scan/zmap4rdns/src/cyclic.h b/rdns_scan/zmap4rdns/src/cyclic.h new file mode 100644 index 0000000..ca31909 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/cyclic.h @@ -0,0 +1,48 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef CYCLIC_H +#define CYCLIC_H + +#include <stdint.h> +#include <stddef.h> + +#include "aesrand.h" + +// Represents a multiplicative cyclic group (Z/pZ)* +typedef struct cyclic_group { + uint64_t prime; // p + uint64_t known_primroot; // Known primitive root of (Z/pZ)* + size_t num_prime_factors; // Length of num_prime_factors + uint64_t prime_factors[10]; // Unique prime factors of (p-1) +} cyclic_group_t; + +// Represents a cycle in a group +typedef struct cycle { + const cyclic_group_t *group; + uint64_t generator; + uint64_t order; + uint32_t offset; +} cycle_t; + +// Get a cyclic_group_t of at least min_size. +// Pointer into static data, do not free(). +const cyclic_group_t *get_group(uint64_t min_size); + +// Generate cycle (find generator and inverse) +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes); + +// Perform the isomorphism from (Z/pZ)+ to (Z/pZ)* +// Given known primitive root of (Z/pZ)* n, with x in (Z/pZ)+, do: +// f(x) = n^x mod p +// +// The isomorphism in the reverse direction is discrete log, and is +// therefore hard. +uint64_t isomorphism(uint64_t additive_elt, const cyclic_group_t *mult_group); + +#endif diff --git a/rdns_scan/zmap4rdns/src/expression.c b/rdns_scan/zmap4rdns/src/expression.c new file mode 100644 index 0000000..12ccd61 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/expression.c @@ -0,0 +1,170 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "expression.h" +#include "fieldset.h" + +#include "../lib/xalloc.h" + +/* Static helper functions */ + +static node_t *alloc_node(); +static int eval_gt_node(node_t *node, fieldset_t *fields); +static int eval_lt_node(node_t *node, fieldset_t *fields); +static int eval_eq_node(node_t *node, fieldset_t *fields); +static int eval_lt_eq_node(node_t *node, fieldset_t *fields); +static int eval_gt_eq_node(node_t *node, fieldset_t *fields); + +static node_t *alloc_node() +{ + node_t *node = xmalloc(sizeof(node_t)); + return node; +} + +static int eval_gt_node(node_t *node, fieldset_t *fields) +{ + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + return (actual > expected); +} + +static int eval_lt_node(node_t *node, fieldset_t *fields) +{ + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + return (actual < expected); +} + +static int eval_eq_node(node_t *node, fieldset_t *fields) +{ + node_t *literal = node->right_child; + int index = node->left_child->value.field.index; + char *expected, *actual; + switch (literal->type) { + case STRING: + expected = literal->value.string_literal; + actual = fs_get_string_by_index(fields, index); + return (strcmp(expected, actual) == 0); + break; + case INT: + return (fs_get_uint64_by_index(fields, index) == + literal->value.int_literal); + break; + default: + printf("wat\n"); + break; + } + return 0; +} + +static int eval_lt_eq_node(node_t *node, fieldset_t *fields) +{ + return !(eval_gt_node(node, fields)); +} + +static int eval_gt_eq_node(node_t *node, fieldset_t *fields) +{ + return !(eval_lt_node(node, fields)); +} + +/* Exposed functions */ + +node_t *make_op_node(enum operation op) +{ + node_t *node = alloc_node(); + node->type = OP; + node->value.op = op; + return node; +} + +node_t *make_field_node(char *fieldname) +{ + node_t *node = alloc_node(); + node->type = FIELD; + node->value.field.fieldname = fieldname; + return node; +} + +node_t *make_string_node(char *literal) +{ + node_t *node = alloc_node(); + node->type = STRING; + node->value.string_literal = literal; + return node; +} + +node_t *make_int_node(int literal) +{ + node_t *node = alloc_node(); + node->type = INT; + node->value.int_literal = literal; + return node; +} + +int evaluate_expression(node_t *root, fieldset_t *fields) +{ + if (!root) + return 1; + switch (root->type) { /* XXX Not sure if runs */ + case FIELD: + case STRING: + case INT: + return 1; + case OP: + break; + } + switch (root->value.op) { + case GT: + return eval_gt_node(root, fields); + case LT: + return eval_lt_node(root, fields); + case EQ: + return eval_eq_node(root, fields); + case NEQ: + return (!eval_eq_node(root, fields)); + case LT_EQ: + return eval_lt_eq_node(root, fields); + case GT_EQ: + return eval_gt_eq_node(root, fields); + case AND: + return (evaluate_expression(root->left_child, fields) && + evaluate_expression(root->right_child, fields)); + case OR: + return (evaluate_expression(root->left_child, fields) || + evaluate_expression(root->right_child, fields)); + } + return 0; +} + +void print_expression(node_t *root) +{ + if (!root) + return; + printf("%s", "( "); + print_expression(root->left_child); + switch (root->type) { + case OP: + printf(" %i ", root->value.op); + break; + case FIELD: + printf(" (%s", root->value.field.fieldname); + break; + case STRING: + printf("%s) ", root->value.string_literal); + break; + case INT: + printf(" %llu) ", (long long unsigned)root->value.int_literal); + break; + default: + break; + } + print_expression(root->right_child); + printf("%s", " )"); +} diff --git a/rdns_scan/zmap4rdns/src/expression.h b/rdns_scan/zmap4rdns/src/expression.h new file mode 100644 index 0000000..d6a5f32 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/expression.h @@ -0,0 +1,54 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_TREE_H +#define ZMAP_TREE_H + +#include "fieldset.h" + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> + +enum operation { GT, LT, EQ, NEQ, AND, OR, LT_EQ, GT_EQ }; + +enum node_type { OP, FIELD, STRING, INT }; + +struct field_id { + int index; + char *fieldname; +}; + +union node_value { + struct field_id field; + char *string_literal; + uint64_t int_literal; + enum operation op; +}; + +typedef struct node_st { + struct node_st *left_child; + struct node_st *right_child; + enum node_type type; + union node_value value; +} node_t; + +node_t *make_op_node(enum operation op); + +node_t *make_field_node(char *fieldname); + +node_t *make_string_node(char *literal); + +node_t *make_int_node(int literal); + +int evaluate_expression(node_t *root, fieldset_t *fields); + +void print_expression(node_t *root); + +#endif /* ZMAP_TREE_H */ diff --git a/rdns_scan/zmap4rdns/src/fieldset.c b/rdns_scan/zmap4rdns/src/fieldset.c new file mode 100644 index 0000000..0bdb92c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/fieldset.c @@ -0,0 +1,409 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "fieldset.h" + +#include <string.h> +#include <stdint.h> +#include <stdlib.h> +#include <assert.h> + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wundef" +#include <unistr.h> +#pragma GCC diagnostic pop + +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len) +{ + if (fds->len + len > MAX_FIELDS) { + log_fatal("fieldset", "out of room in field def set"); + } + fielddef_t *open = &(fds->fielddefs[fds->len]); + memcpy(open, fs, len * sizeof(fielddef_t)); + fds->len += len; +} + +fieldset_t *fs_new_fieldset(fielddefset_t *fds) +{ + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_FIELDSET; + f->fds = fds; + return f; +} + +fieldset_t *fs_new_repeated_field(int type, int free_) +{ + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_REPEATED; + f->inner_type = type; + f->free_ = free_; + return f; +} + +fieldset_t *fs_new_repeated_uint64(void) +{ + return fs_new_repeated_field(FS_UINT64, 0); +} + +fieldset_t *fs_new_repeated_bool(void) +{ + return fs_new_repeated_field(FS_BOOL, 0); +} + +fieldset_t *fs_new_repeated_string(int free_) +{ + return fs_new_repeated_field(FS_STRING, free_); +} + +fieldset_t *fs_new_repeated_binary(int free_) +{ + return fs_new_repeated_field(FS_BINARY, free_); +} + +fieldset_t *fs_new_repeated_fieldset(void) +{ + return fs_new_repeated_field(FS_FIELDSET, 0); +} + +static inline void fs_add_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) +{ + if (fs->len + 1 >= MAX_FIELDS) { + log_fatal("fieldset", "out of room in fieldset"); + } + if (fs->type == FS_REPEATED && fs->inner_type != type) { + log_fatal( + "fieldset", + "object added to repeated field does not match type of repeated field."); + } + field_t *f = &(fs->fields[fs->len]); + // if we have a fieldset definition, then we can validate that the name + // of the field is as expected + if (fs->fds && strcmp(fs->fds->fielddefs[fs->len].name, name)) { + log_fatal("fieldset", "added field (%s) is not next expected field (%s).", + name, fs->fds->fielddefs[fs->len].name); + } + + fs->len++; + f->type = type; + f->name = name; + f->len = len; + f->value = value; + f->free_ = free_; +} + +static void fs_modify_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) +{ + for (int i = 0; i < fs->len; i++) { + if (!strcmp(fs->fields[i].name, name)) { + if (fs->fields[i].free_) { + free(fs->fields[i].value.ptr); + fs->fields[i].value.ptr = NULL; + } + fs->fields[i].type = type; + fs->fields[i].free_ = free_; + fs->fields[i].len = len; + fs->fields[i].value = value; + return; + } + } + // TODO(ZD): We need to test, but this is really unsafe to just add because it + // will all but guarantee that it's in the wrong place + //fs_add_word(fs, name, type, free_, len, value); + log_fatal("fs", "attempting to modify non-existent field"); +} + +static char *sanitize_utf8(const char *buf) +{ + const char *ptr = buf; + + // Count how many errors we encounter + uint32_t i = 0; + // Upper bounds to ensure termination even if u8_check is unsafe + while (i < strlen(buf) && ptr < buf + strlen(buf)) { + ptr = (char *)u8_check((uint8_t *)ptr, strlen(ptr)); + if (ptr == NULL) { + break; + } + + assert(ptr >= buf); + assert(ptr < buf + strlen(buf)); + + ptr++; + i++; + } + + // i is the total number of errors. We need 2 extra bytes for each rune + char *safe_buf = xmalloc(strlen(buf) + i * 2 + 1); + char *safe_ptr = NULL; + memcpy(safe_buf, buf, strlen(buf)); + + // Fix exactly i errors + for (uint32_t j = 0; j < i; j++) { + // Always operate on the working buffer + safe_ptr = + (char *)u8_check((uint8_t *)safe_buf, strlen(safe_buf)); + + // This implies we had less errors than we should. + // This is temporary debug code. + if (safe_ptr == NULL) { + log_warn( + "fieldset", + "UTF8 Sanitization issue. %u errors, fell through iter %u. Orig: %s new: %s", + i, j, buf, safe_buf); + i = j; + break; + } + + // XXX Uncomment when we remove above log_warn. + // assert(safe_ptr != NULL); + assert(safe_ptr >= safe_buf); + assert(safe_ptr < safe_buf + strlen(safe_buf)); + + // Shift the rest of the string by 2 bytes + if (strlen(safe_ptr) > 1) { + memcpy(safe_ptr + 3, safe_ptr + 1, + strlen(safe_ptr + 1)); + } + + // UTF8 replacement rune + safe_ptr[0] = (char)0xef; + safe_ptr[1] = (char)0xbf; + safe_ptr[2] = (char)0xbd; + } + + // We now have a valid utf8 string + assert(u8_check((uint8_t *)safe_buf, strlen(safe_buf)) == NULL); + // We should be null terminated + assert(safe_buf[strlen(buf) + i * 2] == '\0'); + // We should be the right length + assert(strlen(safe_buf) == (strlen(buf) + i * 2)); + + return safe_buf; +} + +void fs_add_null(fieldset_t *fs, const char *name) +{ + field_val_t val = {.ptr = NULL}; + fs_add_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) +{ + if (u8_check((uint8_t *)value, strlen(value)) == NULL) { + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); + } else { + char *safe_value = sanitize_utf8(value); + + if (free_) { + free(value); + } + + field_val_t val = {.ptr = safe_value}; + fs_add_word(fs, name, FS_STRING, 1, strlen(safe_value), val); + } +} + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + if (value) { + fs_add_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) +{ + if (value) { + fs_add_unsafe_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value) +{ + field_val_t val = {.ptr = (char *)value}; + fs_add_word(fs, name, FS_STRING, 0, strlen(value), val); +} + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value) +{ + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_add_bool(fieldset_t *fs, const char *name, int value) +{ + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) +{ + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_BINARY, free_, len, val); +} + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child) +{ + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_FIELDSET, 1, sizeof(void *), val); +} + +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child) +{ + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_REPEATED, 1, sizeof(void *), val); +} + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name) +{ + field_val_t val = {.ptr = NULL}; + fs_modify_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_) +{ + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_modify_constchar(fieldset_t *fs, const char *name, const char *value) +{ + field_val_t val = {.ptr = (char*) value}; + fs_modify_word(fs, name, FS_STRING, 0, strlen(value), val); +} + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value) +{ + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_modify_bool(fieldset_t *fs, const char *name, int value) +{ + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) +{ + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_BINARY, free_, len, val); +} + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index) +{ + return (uint64_t)fs->fields[index].value.num; +} + +char *fs_get_string_by_index(fieldset_t *fs, int index) +{ + return (char *)fs->fields[index].value.ptr; +} + +int fds_get_index_by_name(fielddefset_t *fds, const char *name) +{ + for (int i = 0; i < fds->len; i++) { + if (!strcmp(fds->fielddefs[i].name, name)) { + return i; + } + } + return -1; +} + +void field_free(field_t *f) +{ + if (f->type == FS_FIELDSET || f->type == FS_REPEATED) { + fs_free((fieldset_t *)f->value.ptr); + } else if (f->free_) { + free(f->value.ptr); + } +} + +void fs_free(fieldset_t *fs) +{ + if (!fs) { + return; + } + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + field_free(f); + } + free(fs); +} + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + const char **req, int reqlen) +{ + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", + "unable to allocate memory for translation"); + } + for (int i = 0; i < reqlen; i++) { + int l = fds_get_index_by_name(avail, req[i]); + if (l < 0) { + log_fatal("fieldset", + "specified field (%s) not " + "available in selected " + "probe module.", + req[i]); + } + t->translation[t->len++] = l; + } +} + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail) +{ + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", + "unable to allocate memory for translation"); + } + t->len = avail->len; + for (int i = 0; i < avail->len; i++) { + t->translation[i] = i; + } +} + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) +{ + fieldset_t *retv = fs_new_fieldset(NULL); + if (!retv) { + log_fatal("fieldset", + "unable to allocate space for translated field set"); + } + for (int i = 0; i < t->len; i++) { + int o = t->translation[i]; + memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t)); + } + retv->len = t->len; + return retv; +} diff --git a/rdns_scan/zmap4rdns/src/fieldset.h b/rdns_scan/zmap4rdns/src/fieldset.h new file mode 100644 index 0000000..b85f85c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/fieldset.h @@ -0,0 +1,147 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdint.h> +#include "types.h" + +#ifndef FIELDSET_H +#define FIELDSET_H + +// maximum number of records that can be stored in a fieldset +#define MAX_FIELDS 128 +#define MAX_LIST_LENGTH 255 + +// types of data that can be stored in a field +#define FS_RESERVED 0 +#define FS_STRING 1 +#define FS_UINT64 2 +#define FS_BINARY 3 +#define FS_NULL 4 +#define FS_BOOL 7 +// recursive support +#define FS_FIELDSET 5 +#define FS_REPEATED 6 + +// definition of a field that's provided by a probe module +// these are used so that users can ask at the command-line +// what fields are available for consumption +typedef struct field_def { + const char *name; + const char *type; + const char *desc; +} fielddef_t; + +typedef struct fielddef_set { + fielddef_t fielddefs[MAX_FIELDS]; + int len; +} fielddefset_t; + +typedef union field_val { + void *ptr; + uint64_t num; +} field_val_t; + +// the internal field type used by fieldset +typedef struct field { + const char *name; + int type; + int free_; + size_t len; + field_val_t value; +} field_t; + +// data structure that is populated by the probe module +// and translated into the data structure that's passed +// to the output module +typedef struct fieldset { + int len; + field_t fields[MAX_FIELDS]; + fielddefset_t *fds; + // only used for repeated. + int inner_type; // type of repeated element. e.g., FS_STRING + int type; // REPEATED or FIELDSET + int free_; // should elements be freed +} fieldset_t; + +// we pass a different fieldset to an output module than +// the probe module generates for us because a user may +// only want certain fields and will expect them in a certain +// order. We generate a translated fieldset that contains +// only the fields we want to export to the output module. +// a translation specifies how to efficiently convert the fs +// povided by the probe module to the fs for the output module. +typedef struct translation { + int len; + int translation[MAX_FIELDS]; +} translation_t; + +fieldset_t *fs_new_fieldset(fielddefset_t*); + +fieldset_t *fs_new_repeated_field(int type, int free_); +fieldset_t *fs_new_repeated_uint64(void); +fieldset_t *fs_new_repeated_bool(void); +fieldset_t *fs_new_repeated_string(int free_); +fieldset_t *fs_new_repeated_binary(int free_); +fieldset_t *fs_new_repeated_fieldset(); + +char *fs_get_string_by_index(fieldset_t *fs, int index); + +int fds_get_index_by_name(fielddefset_t *fds, const char *name); + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len); + +void fs_add_null(fieldset_t *fs, const char *name); + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_add_bool(fieldset_t *fs, const char *name, int value); + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value); + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child); +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child); + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name); + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_modify_bool(fieldset_t *fs, const char *name, int value); + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); + +void fs_free(fieldset_t *fs); + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + const char **req, int reqlen); + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail); + +#endif // FIELDSET_H diff --git a/rdns_scan/zmap4rdns/src/filter.c b/rdns_scan/zmap4rdns/src/filter.c new file mode 100644 index 0000000..95e5e4c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/filter.c @@ -0,0 +1,115 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "filter.h" +#include "state.h" +#include "lexer.h" +#include "parser.h" +#include "expression.h" +#include "../lib/logger.h" + +#include <string.h> + +extern int yyparse(); + +node_t *zfilter; + +static int validate_node(node_t *node, fielddefset_t *fields) +{ + int index, found = 0; + if (node->type == OP) { + // These end up getting validated later + if (node->value.op == AND || node->value.op == OR) { + return 1; + } + // Comparison node (=, >, <, etc.) + // Validate that the field (left child) exists in the fieldset + for (index = 0; index < fields->len; index++) { + if (fields->fielddefs[index].name) { + if (strcmp(fields->fielddefs[index].name, + node->left_child->value.field + .fieldname) == 0) { + node->left_child->value.field.index = + index; + found = 1; + break; + } + } + } + if (!found) { + fprintf(stderr, "Field '%s' does not exist\n", + node->left_child->value.field.fieldname); + return 0; + } + // Fieldname is fine, match the type. + switch (node->right_child->type) { + case STRING: + if (strcmp(fields->fielddefs[index].type, "string") == + 0) { + return 1; + } else { + fprintf(stderr, + "Field '%s' is not of type 'string'\n", + fields->fielddefs[index].name); + return 0; + } + case INT: + if (strcmp(fields->fielddefs[index].type, "int") == 0 || + strcmp(fields->fielddefs[index].type, "bool") == + 0) { + return 1; + } else { + fprintf(stderr, + "Field '%s' is not of type 'int'\n", + fields->fielddefs[index].name); + return 0; + } + default: + return 0; + } + } else { + // All non-op nodes are valid + return 1; + } + // Didn't validate + return 0; +} + +int parse_filter_string(char *filter) +{ + YY_BUFFER_STATE buffer_state = yy_scan_string(filter); + int status = yyparse(); + yy_delete_buffer(buffer_state); + if (status) { + // Error + log_error("zmap", "Unable to parse filter string: '%s'", + filter); + return 0; + } + zconf.filter.expression = zfilter; + return 1; +} + +/* + * 0 Valid + * -1 Invalid Field Name + * -2 Type Mismatch + */ +int validate_filter(node_t *root, fielddefset_t *fields) +{ + int valid; + if (!root) { + return 1; + } + valid = validate_node(root, fields); + if (!valid) { + return 0; + } + return (validate_filter(root->left_child, fields) && + validate_filter(root->right_child, fields)); +} diff --git a/rdns_scan/zmap4rdns/src/filter.h b/rdns_scan/zmap4rdns/src/filter.h new file mode 100644 index 0000000..6e23cc5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/filter.h @@ -0,0 +1,23 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_FILTER_H +#define ZMAP_FILTER_H + +#include "expression.h" +#include "fieldset.h" + +struct output_filter { + node_t *expression; +}; + +int parse_filter_string(char *filter); + +int validate_filter(node_t *root, fielddefset_t *fields); + +#endif /* ZMAP_FILTER_H */ diff --git a/rdns_scan/zmap4rdns/src/get_gateway-bsd.h b/rdns_scan/zmap4rdns/src/get_gateway-bsd.h new file mode 100644 index 0000000..a4adc87 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/get_gateway-bsd.h @@ -0,0 +1,199 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_GET_GATEWAY_BSD_H +#define ZMAP_GET_GATEWAY_BSD_H + +#ifdef ZMAP_GET_GATEWAY_LINUX_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include <net/route.h> +#include <net/if.h> +#include <net/if_dl.h> + +#if defined(_SYSTYPE_BSD) + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC_MINOR_ >= 6 +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif + +#include <dnet/os.h> +#include <dnet/eth.h> +#include <dnet/ip.h> +#include <dnet/ip6.h> +#include <dnet/addr.h> +#include <dnet/arp.h> +#endif + +#define ROUNDUP(a) ((a) > 0 ? (1 + (((a)-1) | (sizeof(int) - 1))) : sizeof(int)) +#define UNUSED __attribute__((unused)) + +int get_hw_addr(struct in_addr *gw_ip, UNUSED char *iface, + unsigned char *hw_mac) +{ + arp_t *arp; + struct arp_entry entry; + + if (!gw_ip || !hw_mac) { + return EXIT_FAILURE; + } + + if ((arp = arp_open()) == NULL) { + log_error("get_hw_addr", "failed to open arp table"); + return EXIT_FAILURE; + } + + // Convert gateway ip to dnet struct format + memset(&entry, 0, sizeof(struct arp_entry)); + entry.arp_pa.addr_type = ADDR_TYPE_IP; + entry.arp_pa.addr_bits = IP_ADDR_BITS; + entry.arp_pa.addr_ip = gw_ip->s_addr; + + if (arp_get(arp, &entry) < 0) { + log_debug("get_hw_addr", "failed to fetch arp entry"); + return EXIT_FAILURE; + } else { + log_debug("get_hw_addr", "found ip %s at hw_addr %s", + addr_ntoa(&entry.arp_pa), addr_ntoa(&entry.arp_ha)); + memcpy(hw_mac, &entry.arp_ha.addr_eth, ETHER_ADDR_LEN); + } + arp_close(arp); + return EXIT_SUCCESS; +} + +int get_iface_ip(char *iface, struct in_addr *ip) +{ + assert(iface); + struct ifaddrs *ifaddr, *ifa; + if (getifaddrs(&ifaddr)) { + log_fatal( + "get-iface-ip", + "ZMap is unable able to retrieve a list of available network " + "interfaces: %s. You can manually specify the network interface " + "to use with the \"-i\" flag.", strerror(errno)); + } + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || + ifa->ifa_addr->sa_family != AF_INET) { + continue; + } + if (!strcmp(iface, ifa->ifa_name)) { + struct sockaddr_in *sin = + (struct sockaddr_in *)ifa->ifa_addr; + ip->s_addr = sin->sin_addr.s_addr; + log_debug("get-iface-ip", "IP address found for %s: %s", + iface, inet_ntoa(*ip)); + return EXIT_SUCCESS; + } + } + log_fatal("get-iface-ip", "The specified network interface (\"%s\") does not" + " exist or does not have an assigned IPv4 address.", iface); + return EXIT_FAILURE; +} + +int get_iface_hw_addr(char *iface, unsigned char *hw_mac) +{ + eth_t *e = eth_open(iface); + if (e) { + eth_addr_t eth_addr; + int res = eth_get(e, ð_addr); + log_debug("gateway", "res: %d", res); + if (res == 0) { + memcpy(hw_mac, eth_addr.data, ETHER_ADDR_LEN); + return EXIT_SUCCESS; + } + } + return EXIT_FAILURE; +} + +int _get_default_gw(struct in_addr *gw, char **iface) +{ + char buf[4096]; + struct rt_msghdr *rtm = (struct rt_msghdr *)&buf; + memset(rtm, 0, sizeof(buf)); + int seq = 0x00FF; + rtm->rtm_msglen = sizeof(buf); + rtm->rtm_type = RTM_GET; + rtm->rtm_flags = RTF_GATEWAY; + rtm->rtm_version = RTM_VERSION; + rtm->rtm_seq = seq; + rtm->rtm_addrs = RTA_DST | RTA_IFP; + rtm->rtm_pid = getpid(); + + int fd = socket(PF_ROUTE, SOCK_RAW, 0); + assert(fd > 0); + if (!write(fd, (char *)rtm, sizeof(buf))) { + log_fatal("get-gateway", "Unable to send request to retrieve default" + "gateway MAC address. You will need to manually specify your " + "gateway MAC with the \"-G\" or \"--gateway-mac\" flag."); + } + size_t len; + while (rtm->rtm_type == RTM_GET && + (len = read(fd, rtm, sizeof(buf))) > 0) { + if (len < (int)sizeof(*rtm)) { + return (-1); + } + if (rtm->rtm_type == RTM_GET && rtm->rtm_pid == getpid() && + rtm->rtm_seq == seq) { + if (rtm->rtm_errno) { + errno = rtm->rtm_errno; + return (-1); + } + break; + } + } + + struct sockaddr *sa = (struct sockaddr *)(rtm + 1); + for (int i = 0; i < RTAX_MAX; i++) { + if (rtm->rtm_addrs & (1 << i)) { + if ((1 << i) == RTA_IFP) { + struct sockaddr_dl *sdl = + (struct sockaddr_dl *)sa; + if (!sdl) { + log_fatal("get-gateway", "Unable to parse kernel response to request " + "for gateway MAC address. You will need to manually specify " + "your gateway MAC with the \"-G\" or \"--gateway-mac\" flag."); + } + char *_iface = xmalloc(sdl->sdl_nlen + 1); + memcpy(_iface, sdl->sdl_data, sdl->sdl_nlen); + _iface[sdl->sdl_nlen + 1] = 0; + *iface = _iface; + } + if ((1 << i) == RTA_GATEWAY) { + struct sockaddr_in *sin = + (struct sockaddr_in *)sa; + gw->s_addr = sin->sin_addr.s_addr; + } + // next element + sa = (struct sockaddr *)(ROUNDUP(sa->sa_len) + + (char *)sa); + } + } + close(fd); + return EXIT_SUCCESS; +} + +char *get_default_iface(void) +{ + struct in_addr t; + char *retv = NULL; + _get_default_gw(&t, &retv); + return retv; +} + +int get_default_gw(struct in_addr *gw, UNUSED char *iface) +{ + char *_iface = NULL; + _get_default_gw(gw, &_iface); + return EXIT_SUCCESS; +} + +#endif /* ZMAP_GET_GATEWAY_BSD_H */ diff --git a/rdns_scan/zmap4rdns/src/get_gateway-linux.h b/rdns_scan/zmap4rdns/src/get_gateway-linux.h new file mode 100644 index 0000000..629c571 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/get_gateway-linux.h @@ -0,0 +1,311 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_GET_GATEWAY_LINUX_H +#define ZMAP_GET_GATEWAY_LINUX_H + +#ifdef ZMAP_GET_GATEWAY_BSD_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include <sys/ioctl.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> +#include <arpa/inet.h> +#include <pcap/pcap.h> + +#define GW_BUFFER_SIZE 64000 + +char *get_default_iface(void) +{ + char errbuf[PCAP_ERRBUF_SIZE]; + char *iface = pcap_lookupdev(errbuf); + if (iface == NULL) { + log_fatal("send", "ZMap could not detect your default network interface. " + "You likely do not privileges to open a raw packet socket. " + "Are you running as root or with the CAP_NET_RAW capability? If you are, you " + "may need to manually set interface using the \"-i\" flag."); + } + return iface; +} + +int read_nl_sock(int sock, char *buf, int buf_len) +{ + int msg_len = 0; + char *pbuf = buf; + do { + int len = recv(sock, pbuf, buf_len - msg_len, 0); + if (len <= 0) { + log_debug("get-gw", "recv failed: %s", strerror(errno)); + return -1; + } + struct nlmsghdr *nlhdr = (struct nlmsghdr *)pbuf; + if (NLMSG_OK(nlhdr, ((unsigned int)len)) == 0 || + nlhdr->nlmsg_type == NLMSG_ERROR) { + log_debug("get-gw", "recv failed: %s", strerror(errno)); + return -1; + } + if (nlhdr->nlmsg_type == NLMSG_DONE) { + break; + } else { + msg_len += len; + pbuf += len; + } + if ((nlhdr->nlmsg_flags & NLM_F_MULTI) == 0) { + break; + } + } while (1); + return msg_len; +} + +int send_nl_req(uint16_t msg_type, uint32_t seq, void *payload, + uint32_t payload_len) +{ + int sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE); + if (sock < 0) { + log_error("get-gw", "unable to get socket: %s", + strerror(errno)); + return -1; + } + if (NLMSG_SPACE(payload_len) < payload_len) { + close(sock); + // Integer overflow + return -1; + } + struct nlmsghdr *nlmsg; + nlmsg = xmalloc(NLMSG_SPACE(payload_len)); + + memset(nlmsg, 0, NLMSG_SPACE(payload_len)); + memcpy(NLMSG_DATA(nlmsg), payload, payload_len); + nlmsg->nlmsg_type = msg_type; + nlmsg->nlmsg_len = NLMSG_LENGTH(payload_len); + nlmsg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST; + nlmsg->nlmsg_seq = seq; + nlmsg->nlmsg_pid = getpid(); + + if (send(sock, nlmsg, nlmsg->nlmsg_len, 0) < 0) { + log_error("get-gw", "failure sending: %s", strerror(errno)); + return -1; + } + free(nlmsg); + return sock; +} + +int get_hw_addr(struct in_addr *gw_ip, char *iface, unsigned char *hw_mac) +{ + struct ndmsg req; + memset(&req, 0, sizeof(struct ndmsg)); + + if (!gw_ip || !hw_mac) { + return -1; + } + // Send RTM_GETNEIGH request + req.ndm_family = AF_INET; + req.ndm_ifindex = if_nametoindex(iface); + req.ndm_state = NUD_REACHABLE; + req.ndm_type = NDA_LLADDR; + + int sock = send_nl_req(RTM_GETNEIGH, 1, &req, sizeof(req)); + // Read responses + char *buf = xmalloc(GW_BUFFER_SIZE); + int nl_len = read_nl_sock(sock, buf, GW_BUFFER_SIZE); + if (nl_len <= 0) { + free(buf); + return -1; + } + // Parse responses + struct nlmsghdr *nlhdr = (struct nlmsghdr *)buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + unsigned char mac[6]; + struct in_addr dst_ip; + int correct_ip = 0; + + rt_msg = (struct rtmsg *)NLMSG_DATA(nlhdr); + if ((rt_msg->rtm_family != AF_INET)) { + free(buf); + return -1; + } + rt_attr = (struct rtattr *)RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case NDA_LLADDR: + if (RTA_PAYLOAD(rt_attr) != IFHWADDRLEN) { + // could be using a VPN + log_fatal( + "get_gateway", + "Unexpected hardware address length (%d)." + " If you are using a VPN, supply the --iplayer flag (and provide an" + " interface via -i)", + RTA_PAYLOAD(rt_attr)); + } + memcpy(mac, RTA_DATA(rt_attr), IFHWADDRLEN); + break; + case NDA_DST: + if (RTA_PAYLOAD(rt_attr) != sizeof(dst_ip)) { + // could be using a VPN + log_fatal( + "get_gateway", + "Unexpected IP address length (%d)." + " If you are using a VPN, supply the --iplayer flag" + " (and provide an interface via -i)", + RTA_PAYLOAD(rt_attr)); + } + memcpy(&dst_ip, RTA_DATA(rt_attr), + sizeof(dst_ip)); + if (memcmp(&dst_ip, gw_ip, sizeof(dst_ip)) == + 0) { + correct_ip = 1; + } + break; + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + if (correct_ip) { + memcpy(hw_mac, mac, IFHWADDRLEN); + free(buf); + return 0; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + free(buf); + return -1; +} + +// gw and iface[IF_NAMESIZE] MUST be allocated +int _get_default_gw(struct in_addr *gw, char *iface) +{ + struct rtmsg req; + unsigned int nl_len; + char buf[8192]; + struct nlmsghdr *nlhdr; + + if (!gw || !iface) { + return -1; + } + + // Send RTM_GETROUTE request + memset(&req, 0, sizeof(req)); + int sock = send_nl_req(RTM_GETROUTE, 0, &req, sizeof(req)); + + // Read responses + nl_len = read_nl_sock(sock, buf, sizeof(buf)); + if (nl_len <= 0) { + return -1; + } + + // Parse responses + nlhdr = (struct nlmsghdr *)buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + int has_gw = 0; + + rt_msg = (struct rtmsg *)NLMSG_DATA(nlhdr); + + // There could be multiple routing tables. Loop until we find the + // correct one. + if ((rt_msg->rtm_family != AF_INET) || + (rt_msg->rtm_table != RT_TABLE_MAIN)) { + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + continue; + } + + rt_attr = (struct rtattr *)RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case RTA_OIF: + if_indextoname(*(int *)RTA_DATA(rt_attr), + iface); + break; + case RTA_GATEWAY: + gw->s_addr = *(unsigned int *)RTA_DATA(rt_attr); + has_gw = 1; + break; + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + + if (has_gw) { + return 0; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + return -1; +} + +int get_default_gw(struct in_addr *gw, char *iface) +{ + char _iface[IF_NAMESIZE]; + memset(_iface, 0, IF_NAMESIZE); + + _get_default_gw(gw, _iface); + if (strcmp(iface, _iface)) { + log_fatal( + "get-gateway", + "The specified network (\"%s\") does not match " + "the interface associated with the default gateway (%s). You will " + "need to manually specify the MAC address of your gateway using " + "the \"--gateway-mac\" flag.", iface, _iface); + } + return EXIT_SUCCESS; +} + +int get_iface_ip(char *iface, struct in_addr *ip) +{ + int sock; + struct ifreq ifr; + memset(&ifr, 0, sizeof(struct ifreq)); + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock < 0) { + log_fatal("get-iface-ip", "failure opening socket: %s", + strerror(errno)); + } + ifr.ifr_addr.sa_family = AF_INET; + strncpy(ifr.ifr_name, iface, IFNAMSIZ - 1); + + if (ioctl(sock, SIOCGIFADDR, &ifr) < 0) { + close(sock); + log_fatal("get-iface-ip", "Unable to automatically identify the correct " + "source address for %s interface. ioctl failure: %s. " + "If this is the unexpected interface, you can manually specify " + "the correct interface with \"-i\" flag. If this is the correct " + "interface, you likely need to manually specify the source IP " + "address to use with the \"-S\" flag.", iface, strerror(errno)); + } + ip->s_addr = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr.s_addr; + close(sock); + return EXIT_SUCCESS; +} + +int get_iface_hw_addr(char *iface, unsigned char *hw_mac) +{ + int s; + struct ifreq buffer; + + // Load the hwaddr from a dummy socket + s = socket(PF_INET, SOCK_DGRAM, 0); + if (s < 0) { + log_error("get_iface_hw_addr", "Unable to open socket: %s", + strerror(errno)); + return EXIT_FAILURE; + } + memset(&buffer, 0, sizeof(buffer)); + strncpy(buffer.ifr_name, iface, IFNAMSIZ); + ioctl(s, SIOCGIFHWADDR, &buffer); + close(s); + memcpy(hw_mac, buffer.ifr_hwaddr.sa_data, 6); + return EXIT_SUCCESS; +} + +#endif /* ZMAP_GET_GATEWAY_LINUX_H */ diff --git a/rdns_scan/zmap4rdns/src/get_gateway.c b/rdns_scan/zmap4rdns/src/get_gateway.c new file mode 100644 index 0000000..87abfcc --- /dev/null +++ b/rdns_scan/zmap4rdns/src/get_gateway.c @@ -0,0 +1,30 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <assert.h> +#include <string.h> + +#include <pcap/pcap.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +#include <sys/ioctl.h> + +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#include "get_gateway-bsd.h" +#else // (linux) +#include "get_gateway-linux.h" +#endif diff --git a/rdns_scan/zmap4rdns/src/get_gateway.h b/rdns_scan/zmap4rdns/src/get_gateway.h new file mode 100644 index 0000000..2df448c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/get_gateway.h @@ -0,0 +1,20 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef GET_GATEWAY_H +#define GET_GATEWAY_H + +#include <netinet/in.h> + +int get_hw_addr(struct in_addr *gw_ip, char *iface, unsigned char *hw_mac); +int get_default_gw(struct in_addr *gw, char *iface); +int get_iface_ip(char *iface, struct in_addr *ip); +int get_iface_hw_addr(char *iface, unsigned char *hw_mac); +char *get_default_iface(void); + +#endif diff --git a/rdns_scan/zmap4rdns/src/iterator.c b/rdns_scan/zmap4rdns/src/iterator.c new file mode 100644 index 0000000..8821031 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/iterator.c @@ -0,0 +1,128 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <assert.h> +#include <pthread.h> +#include <stdint.h> +#include <time.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +#include "iterator.h" + +#include "aesrand.h" +#include "shard.h" +#include "state.h" + +struct iterator { + cycle_t cycle; + uint8_t num_threads; + shard_t *thread_shards; + uint8_t *complete; + pthread_mutex_t mutex; + uint32_t curr_threads; +}; + +void shard_complete(uint8_t thread_id, void *arg) +{ + iterator_t *it = (iterator_t *)arg; + assert(thread_id < it->num_threads); + pthread_mutex_lock(&it->mutex); + it->complete[thread_id] = 1; + it->curr_threads--; + shard_t *s = &it->thread_shards[thread_id]; + zsend.packets_sent += s->state.packets_sent; + zsend.hosts_scanned += s->state.hosts_scanned; + zsend.blocklisted += s->state.hosts_blocklisted; + zsend.allowlisted += s->state.hosts_allowlisted; + zsend.sendto_failures += s->state.packets_failed; + uint8_t done = 1; + for (uint8_t i = 0; done && (i < it->num_threads); ++i) { + done = done && it->complete[i]; + } + if (done) { + zsend.finish = now(); + zsend.complete = 1; + zsend.first_scanned = it->thread_shards[0].state.first_scanned; + } + pthread_mutex_unlock(&it->mutex); +} + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards) +{ + uint64_t num_addrs = blocklist_count_allowed(); + uint64_t group_min_size = num_addrs; + if (zconf.list_of_ips_filename) { + log_debug("send", "forcing max group size for compatibility with -I"); + group_min_size = 0xFFFFFFFF; + } + iterator_t *it = xmalloc(sizeof(struct iterator)); + const cyclic_group_t *group = get_group(group_min_size); + if (num_addrs > (1LL << 32)) { + zsend.max_index = 0xFFFFFFFF; + } else { + zsend.max_index = (uint32_t)num_addrs; + } + log_debug("iterator", "max index %u", zsend.max_index); + it->cycle = make_cycle(group, zconf.aes); + it->num_threads = num_threads; + it->curr_threads = num_threads; + it->thread_shards = xcalloc(num_threads, sizeof(shard_t)); + it->complete = xcalloc(it->num_threads, sizeof(uint8_t)); + pthread_mutex_init(&it->mutex, NULL); + for (uint8_t i = 0; i < num_threads; ++i) { + shard_init(&it->thread_shards[i], shard, num_shards, i, + num_threads, zsend.max_targets, &it->cycle, + shard_complete, it); + } + zconf.generator = it->cycle.generator; + return it; +} + +uint64_t iterator_get_sent(iterator_t *it) +{ + uint64_t sent = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + sent += it->thread_shards[i].state.packets_sent; + } + return sent; +} + +uint64_t iterator_get_iterations(iterator_t *it) +{ + uint64_t iterations = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + iterations += it->thread_shards[i].iterations; + } + return iterations; +} + +uint32_t iterator_get_fail(iterator_t *it) +{ + uint32_t fails = 0; + for (uint8_t i = 0; i < it->num_threads; ++i) { + fails += it->thread_shards[i].state.packets_failed; + } + return fails; +} + +shard_t *get_shard(iterator_t *it, uint8_t thread_id) +{ + assert(thread_id < it->num_threads); + return &it->thread_shards[thread_id]; +} + +uint32_t iterator_get_curr_send_threads(iterator_t *it) +{ + assert(it); + return it->curr_threads; +} diff --git a/rdns_scan/zmap4rdns/src/iterator.h b/rdns_scan/zmap4rdns/src/iterator.h new file mode 100644 index 0000000..383a2f0 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/iterator.h @@ -0,0 +1,33 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_ITERATOR_H +#define ZMAP_ITERATOR_H + +#include <stdint.h> + +#include "../lib/includes.h" + +#include "aesrand.h" +#include "cyclic.h" +#include "shard.h" + +typedef struct iterator iterator_t; + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards); + +uint64_t iterator_get_sent(iterator_t *it); +uint64_t iterator_get_iterations(iterator_t *it); +uint32_t iterator_get_fail(iterator_t *it); + +uint32_t iterator_get_curr_send_threads(iterator_t *it); + +shard_t *get_shard(iterator_t *it, uint8_t thread_id); + +#endif /* ZMAP_ITERATOR_H */ diff --git a/rdns_scan/zmap4rdns/src/lexer.l b/rdns_scan/zmap4rdns/src/lexer.l new file mode 100644 index 0000000..48e0ad2 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/lexer.l @@ -0,0 +1,36 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +%{ +#pragma GCC diagnostic ignored "-Wredundant-decls" +#pragma GCC diagnostic ignored "-Wmissing-noreturn" + +#include <string.h> +#include "parser.h" + +%} + +%option noinput +%option nounput +%% +[0-9]+ yylval.int_literal = (uint64_t) atoll(yytext); return T_NUMBER; +\n /* Ignore end of line */ +[ \t]+ /* Ignore whitespace */ +!= return T_NOT_EQ; +>= return T_GT_EQ; +"<=" return T_LT_EQ; +&& return T_AND; +"||" return T_OR; += return '='; +">" return '>'; +"<" return '<'; +"(" return '('; +")" return ')'; +[a-zA-Z][-_a-zA-Z0-9]+ yylval.string_literal = strdup(yytext); return T_FIELD; + +%% diff --git a/rdns_scan/zmap4rdns/src/monitor.c b/rdns_scan/zmap4rdns/src/monitor.c new file mode 100644 index 0000000..63369a5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/monitor.c @@ -0,0 +1,487 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// module responsible for printing on-screen updates during the scan process + +#include "monitor.h" + +#define __STDC_FORMAT_MACROS +#include <inttypes.h> + +#include <assert.h> +#include <errno.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <time.h> +#include <unistd.h> + +#include "iterator.h" +#include "recv.h" +#include "state.h" + +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#define UPDATE_INTERVAL 1 // seconds +#define NUMBER_STR_LEN 20 +#define WARMUP_PERIOD 5 +#define MIN_HITRATE_TIME_WINDOW 5 // seconds + +// internal monitor status that is used to track deltas +typedef struct internal_scan_status { + double last_now; + uint64_t last_sent; + uint64_t last_tried_sent; + uint32_t last_send_failures; + uint32_t last_recv_net_success; + uint32_t last_recv_app_success; + uint32_t last_recv_total; + uint32_t last_pcap_drop; + double min_hitrate_start; +} int_status_t; + +// exportable status information that can be printed to screen +typedef struct export_scan_status { + uint64_t total_sent; + uint64_t total_tried_sent; + uint32_t recv_success_unique; + uint32_t app_recv_success_unique; + uint64_t total_recv; + uint32_t complete; + uint32_t send_threads; + double percent_complete; + + double hitrate; // network, e.g. SYN-ACK vs RST + double + app_hitrate; // application level, e.g. DNS response versus correct + // lookup. + + double send_rate; + char send_rate_str[NUMBER_STR_LEN]; + double send_rate_avg; + char send_rate_avg_str[NUMBER_STR_LEN]; + + double recv_rate; + char recv_rate_str[NUMBER_STR_LEN]; + double recv_avg; + char recv_avg_str[NUMBER_STR_LEN]; + double recv_total_rate; + double recv_total_avg; + + double app_success_rate; + char app_success_rate_str[NUMBER_STR_LEN]; + double app_success_avg; + char app_success_avg_str[NUMBER_STR_LEN]; + + uint32_t pcap_drop; + uint32_t pcap_ifdrop; + uint32_t pcap_drop_total; + char pcap_drop_total_str[NUMBER_STR_LEN]; + double pcap_drop_last; + char pcap_drop_last_str[NUMBER_STR_LEN]; + double pcap_drop_avg; + char pcap_drop_avg_str[NUMBER_STR_LEN]; + + uint32_t time_remaining; + char time_remaining_str[NUMBER_STR_LEN]; + uint32_t time_past; + char time_past_str[NUMBER_STR_LEN]; + + uint32_t fail_total; + double fail_avg; + double fail_last; + float seconds_under_min_hitrate; + +} export_status_t; + +static FILE *status_fd = NULL; + +// find minimum of an array of doubles +static double min_d(double array[], int n) +{ + double value = INFINITY; + for (int i = 0; i < n; i++) { + if (array[i] < value) { + value = array[i]; + } + } + return value; +} + +// estimate time remaining time based on config and state +double compute_remaining_time(double age, uint64_t packets_sent, uint64_t iterations) +{ + if (!zsend.complete) { + double remaining[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + if (zsend.list_of_ips_pbm) { + // Estimate progress using group iterations + double done = (double) iterations / + ((uint64_t)0xFFFFFFFFU / + zconf.total_shards); + remaining[0] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + if (zsend.max_targets) { + double done = + (double)packets_sent / + ((uint64_t)zsend.max_targets * zconf.packet_streams / + zconf.total_shards); + remaining[1] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + if (zconf.max_runtime) { + remaining[2] = + (zconf.max_runtime - age) + zconf.cooldown_secs; + } + if (zconf.max_results) { + double done = + (double)zrecv.filter_success / zconf.max_results; + remaining[3] = (1. - done) * (age / done); + } + if (zsend.max_index) { + double done = (double)packets_sent / + (zsend.max_index * zconf.packet_streams / + zconf.total_shards); + remaining[4] = + (1. - done) * (age / done) + zconf.cooldown_secs; + } + return min_d(remaining, sizeof(remaining) / sizeof(double)); + } else { + return zconf.cooldown_secs - (now() - zsend.finish); + } +} + +static void update_pcap_stats(pthread_mutex_t *recv_ready_mutex) +{ + // ask pcap for fresh values + pthread_mutex_lock(recv_ready_mutex); + recv_update_stats(); + pthread_mutex_unlock(recv_ready_mutex); +} + +static void export_stats(int_status_t *intrnl, export_status_t *exp, + iterator_t *it) +{ + uint64_t total_sent = iterator_get_sent(it); + uint64_t total_iterations = iterator_get_iterations(it); + uint32_t total_fail = iterator_get_fail(it); + uint64_t total_recv = zrecv.pcap_recv; + uint64_t recv_success = zrecv.filter_success; + uint32_t app_success = zrecv.app_success_unique; + double cur_time = now(); + double age = cur_time - zsend.start; // time of entire scan + // time since the last time we updated + double delta = cur_time - intrnl->last_now; + double remaining_secs = compute_remaining_time(age, total_sent, total_iterations); + + // export amount of time the scan has been running + if (age < WARMUP_PERIOD) { + exp->time_remaining_str[0] = '\0'; + } else { + char buf[20]; + time_string(ceil(remaining_secs), 1, buf, sizeof(buf)); + snprintf(exp->time_remaining_str, NUMBER_STR_LEN, " (%s left)", + buf); + } + exp->time_past = age; + exp->time_remaining = remaining_secs; + time_string((int)age, 0, exp->time_past_str, NUMBER_STR_LEN); + + // export recv statistics + exp->recv_rate = + ceil((recv_success - intrnl->last_recv_net_success) / delta); + number_string(exp->recv_rate, exp->recv_rate_str, NUMBER_STR_LEN); + exp->recv_avg = recv_success / age; + number_string(exp->recv_avg, exp->recv_avg_str, NUMBER_STR_LEN); + exp->recv_total_rate = (total_recv - intrnl->last_recv_total) / delta; + exp->recv_total_avg = total_recv / age; + + // application level statistics + if (zconf.fsconf.app_success_index >= 0) { + exp->app_success_rate = + (app_success - intrnl->last_recv_app_success) / delta; + number_string(exp->app_success_rate, exp->app_success_rate_str, + NUMBER_STR_LEN); + exp->app_success_avg = (app_success / age); + number_string(exp->app_success_avg, exp->app_success_avg_str, + NUMBER_STR_LEN); + } + + if (!total_sent) { + exp->hitrate = 0; + exp->app_hitrate = 0; + } else { + exp->hitrate = recv_success * 100.0 / total_sent; + exp->app_hitrate = app_success * 100.0 / total_sent; + } + + if (age > WARMUP_PERIOD && exp->hitrate < zconf.min_hitrate) { + if (fabs(intrnl->min_hitrate_start) < .00001) { + intrnl->min_hitrate_start = cur_time; + } + } else { + intrnl->min_hitrate_start = 0.0; + } + if (fabs(intrnl->min_hitrate_start) < .00001) { + exp->seconds_under_min_hitrate = 0; + } else { + exp->seconds_under_min_hitrate = + cur_time - intrnl->min_hitrate_start; + } + if (!zsend.complete) { + exp->send_rate = ceil((total_sent - intrnl->last_sent) / delta); + number_string(exp->send_rate, exp->send_rate_str, + NUMBER_STR_LEN); + exp->send_rate_avg = total_sent / age; + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + } else { + exp->send_rate_avg = total_sent / (zsend.finish - zsend.start); + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + } + // export other pre-calculated values + exp->total_sent = total_sent; + exp->total_tried_sent = total_iterations; + exp->percent_complete = 100. * age / (age + remaining_secs); + exp->recv_success_unique = recv_success; + exp->app_recv_success_unique = app_success; + exp->total_recv = total_recv; + exp->complete = zsend.complete; + + // pcap dropped packets + exp->pcap_drop = zrecv.pcap_drop; + exp->pcap_ifdrop = zrecv.pcap_ifdrop; + exp->pcap_drop_total = exp->pcap_drop + exp->pcap_ifdrop; + exp->pcap_drop_last = + (exp->pcap_drop_total - intrnl->last_pcap_drop) / delta; + exp->pcap_drop_avg = exp->pcap_drop_total / age; + number_string(exp->pcap_drop_total, exp->pcap_drop_total_str, + NUMBER_STR_LEN); + number_string(exp->pcap_drop_last, exp->pcap_drop_last_str, + NUMBER_STR_LEN); + number_string(exp->pcap_drop_avg, exp->pcap_drop_avg_str, + NUMBER_STR_LEN); + + zsend.sendto_failures = total_fail; + exp->fail_total = zsend.sendto_failures; + exp->fail_last = (exp->fail_total - intrnl->last_send_failures) / delta; + exp->fail_avg = exp->fail_total / age; + + // misc + exp->send_threads = iterator_get_curr_send_threads(it); + + // Update internal stats + intrnl->last_now = cur_time; + intrnl->last_sent = exp->total_sent; + intrnl->last_recv_net_success = exp->recv_success_unique; + intrnl->last_recv_app_success = exp->app_recv_success_unique; + intrnl->last_pcap_drop = exp->pcap_drop_total; + intrnl->last_send_failures = exp->fail_total; + intrnl->last_recv_total = exp->total_recv; +} + +static void log_drop_warnings(export_status_t *exp) +{ + if (exp->pcap_drop_last / exp->recv_rate > 0.05) { + log_warn("monitor", + "Dropped %.0f packets in the last second, (%u total " + "dropped (pcap: %u + iface: %u))", + exp->pcap_drop_last, exp->pcap_drop_total, + exp->pcap_drop, exp->pcap_ifdrop); + } + if (exp->fail_last / exp->send_rate > 0.01) { + log_warn("monitor", + "Failed to send %.0f packets/sec (%u total failures)", + exp->fail_last, exp->fail_total); + } +} + +static void onscreen_appsuccess(export_status_t *exp) +{ + // this when probe module handles application-level success rates + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; sent: %"PRIu64" %sp/s (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "app success: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_str, exp->send_rate_avg_str, + exp->recv_success_unique, exp->recv_rate_str, + exp->recv_avg_str, exp->app_recv_success_unique, + exp->app_success_rate_str, exp->app_success_avg_str, + exp->pcap_drop_last_str, exp->pcap_drop_avg_str, + exp->hitrate, exp->app_hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; sent: %"PRIu64" done (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "app success: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, + exp->app_recv_success_unique, exp->app_success_rate_str, + exp->app_success_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate, exp->app_hitrate); + } +} + +static void onscreen_generic(export_status_t *exp) +{ + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; send: %"PRIu64" %sp/s (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_str, exp->send_rate_avg_str, + exp->recv_success_unique, exp->recv_rate_str, + exp->recv_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; send: %"PRIu64" done (%sp/s avg); " + "recv: %u %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, + exp->pcap_drop_last_str, exp->pcap_drop_avg_str, + exp->hitrate); + } + fflush(stderr); +} + +static FILE *init_status_update_file(char *path) +{ + FILE *f = fopen(path, "w"); + if (!f) { + log_fatal("csv", "could not open status updates file (%s): %s", + zconf.status_updates_file, strerror(errno)); + } + log_debug("monitor", "status updates CSV will be saved to %s", + zconf.status_updates_file); + fprintf( + f, + "real-time,time-elapsed,time-remaining," + "percent-complete,hit-rate,active-send-threads," + "sent-total,sent-last-one-sec,sent-avg-per-sec," + "recv-success-total,recv-success-last-one-sec,recv-success-avg-per-sec," + "recv-total,recv-total-last-one-sec,recv-total-avg-per-sec," + "pcap-drop-total,drop-last-one-sec,drop-avg-per-sec," + "sendto-fail-total,sendto-fail-last-one-sec,sendto-fail-avg-per-sec\n"); + fflush(f); + return f; +} + +static void update_status_updates_file(export_status_t *exp, FILE *f) +{ + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%Y-%m-%d %H:%M:%S", ptm); + + fprintf(f, + "%s,%u,%u," + "%f,%f,%u," + "%"PRIu64",%.0f,%.0f," + "%u,%.0f,%.0f," + "%"PRIu64",%.0f,%.0f," + "%u,%.0f,%.0f," + "%u,%.0f,%.0f\n", + timestamp, exp->time_past, exp->time_remaining, + exp->percent_complete, exp->hitrate, exp->send_threads, + exp->total_sent, exp->send_rate, exp->send_rate_avg, + exp->recv_success_unique, exp->recv_rate, exp->recv_avg, + exp->total_recv, exp->recv_total_rate, exp->recv_total_avg, + exp->pcap_drop_total, exp->pcap_drop_last, exp->pcap_drop_avg, + exp->fail_total, exp->fail_last, exp->fail_avg); + fflush(f); +} + +static inline void check_min_hitrate(export_status_t *exp) +{ + if (exp->seconds_under_min_hitrate >= MIN_HITRATE_TIME_WINDOW) { + log_fatal("monitor", + "hitrate below %.0f for %.0f seconds. aborting scan.", + zconf.min_hitrate, exp->seconds_under_min_hitrate); + } +} + +static inline void check_max_sendto_failures(export_status_t *exp) +{ + if (zconf.max_sendto_failures >= 0 && + exp->fail_total > (uint32_t)zconf.max_sendto_failures) { + log_fatal("monitor", + "maximum number of sendto failures (%i) exceeded", + zconf.max_sendto_failures); + } +} + +void monitor_init(void) +{ + if (zconf.status_updates_file) { + status_fd = init_status_update_file(zconf.status_updates_file); + assert(status_fd); + } +} + +void monitor_run(iterator_t *it, pthread_mutex_t *lock) +{ + int_status_t *internal_status = xmalloc(sizeof(int_status_t)); + export_status_t *export_status = xmalloc(sizeof(export_status_t)); + + while (!(zsend.complete && zrecv.complete)) { + update_pcap_stats(lock); + export_stats(internal_status, export_status, it); + log_drop_warnings(export_status); + check_min_hitrate(export_status); + check_max_sendto_failures(export_status); + if (!zconf.quiet) { + lock_file(stderr); + if (zconf.fsconf.app_success_index >= 0) { + onscreen_appsuccess(export_status); + } else { + onscreen_generic(export_status); + } + unlock_file(stderr); + } + if (status_fd) { + update_status_updates_file(export_status, status_fd); + } + sleep(UPDATE_INTERVAL); + } + if (!zconf.quiet) { + lock_file(stderr); + fflush(stderr); + unlock_file(stderr); + } + if (status_fd) { + fflush(status_fd); + fclose(status_fd); + } +} diff --git a/rdns_scan/zmap4rdns/src/monitor.h b/rdns_scan/zmap4rdns/src/monitor.h new file mode 100644 index 0000000..6aa564c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/monitor.h @@ -0,0 +1,18 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <pthread.h> +#include "iterator.h" + +#ifndef MONITOR_H +#define MONITOR_H + +void monitor_run(iterator_t *it, pthread_mutex_t *lock); +void monitor_init(void); + +#endif diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_csv.c b/rdns_scan/zmap4rdns/src/output_modules/module_csv.c new file mode 100644 index 0000000..028a088 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_csv.c @@ -0,0 +1,234 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <assert.h> +#include <inttypes.h> + +#include "../../lib/logger.h" +#include "../fieldset.h" + +#include "output_modules.h" + +static FILE *file = NULL; +static int saddr_index = -1; +static int data_index = -1; + +int csv_init(struct state_conf *conf, const char **fields, int fieldlens) +{ + assert(conf); + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal( + "csv", + "could not open CSV output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + } else { + file = stdout; + log_debug("csv", "no output file selected, will use stdout"); + } + if (!conf->no_header_row) { + log_debug("csv", "more than one field, will add headers"); + for (int i = 0; i < fieldlens; i++) { + if (i) { + fprintf(file, ","); + } + fprintf(file, "%s", fields[i]); + } + fprintf(file, "\n"); + } + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +int csv_close(__attribute__((unused)) struct state_conf *c, + __attribute__((unused)) struct state_send *s, + __attribute__((unused)) struct state_recv *r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +static void hex_encode(FILE *f, unsigned char *readbuf, size_t len) +{ + for (size_t i = 0; i < len; i++) { + fprintf(f, "%02x", readbuf[i]); + } + check_and_log_file_error(f, "csv"); +} + +int csv_process(fieldset_t *fs) +{ + if (!file) { + return EXIT_SUCCESS; + } + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (i) { + fprintf(file, ","); + } + if (f->type == FS_STRING) { + if (strchr((char *)f->value.ptr, ',')) { + fprintf(file, "\"%s\"", (char *)f->value.ptr); + } else { + fprintf(file, "%s", (char *)f->value.ptr); + } + } else if (f->type == FS_UINT64) { + fprintf(file, "%" PRIu64, (uint64_t)f->value.num); + } else if (f->type == FS_BOOL) { + fprintf(file, "%" PRIi32, (int)f->value.num); + } else if (f->type == FS_BINARY) { + hex_encode(file, (unsigned char *)f->value.ptr, f->len); + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("csv", "received unknown output type"); + } + } + fprintf(file, "\n"); + fflush(file); + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +output_module_t module_csv_file = { + .name = "csv", + .init = &csv_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &csv_close, + .process_ip = &csv_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = + "Outputs one or more output fields as a comma-delimited file. By default, the " + "probe module does not filter out duplicates or limit to successful fields, " + "but rather includes all received packets. Fields can be controlled by " + "setting --output-fields. Filtering out failures and duplicate packets can " + "be achieved by setting an --output-filter." +}; + + +int csv4rdns_init(struct state_conf *conf, const char **fields, int fieldlens) +{ + assert(conf); + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal( + "csv", + "could not open CSV output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + } else { + file = stdout; + log_debug("csv", "no output file selected, will use stdout"); + } + if (!conf->no_header_row) { + // save the saddr field index and data field index + for (int i = 0; i < fieldlens; i++) { + if(!strcmp(fields[i], "saddr")) + saddr_index = i; + else if(!strcmp(fields[i], "data")){ + data_index = i; + } + } + } + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +int csv4rdns_close(__attribute__((unused)) struct state_conf *c, + __attribute__((unused)) struct state_send *s, + __attribute__((unused)) struct state_recv *r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +// confirm the RA bit in the DNS reponse +int RA_confirm(unsigned char *readbuf, size_t len) +{ + if(len < 12) + return 0; + if((readbuf[2] == 0x81) && (readbuf[3] == 0x80)) + return 1; + else + return 0; +} + +int csv4rdns_process(fieldset_t *fs) +{ + if (!file || data_index == -1 || saddr_index == -1) { + return EXIT_SUCCESS; + } + // if the RA bit is 1, print the saddr + field_t *f = &(fs->fields[data_index]); + if(RA_confirm((unsigned char *)f->value.ptr, f->len)){ + f = &(fs->fields[saddr_index]); + if (f->type == FS_STRING) { + if (strchr((char *)f->value.ptr, ',')) { + fprintf(file, "\"%s\"", (char *)f->value.ptr); + } else { + fprintf(file, "%s", (char *)f->value.ptr); + } + } else if (f->type == FS_UINT64) { + fprintf(file, "%" PRIu64, (uint64_t)f->value.num); + } else if (f->type == FS_BOOL) { + fprintf(file, "%" PRIi32, (int)f->value.num); + } else if (f->type == FS_BINARY) { + hex_encode(file, (unsigned char *)f->value.ptr, f->len); + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("csv", "received unknown output type"); + } + fprintf(file, "\n"); + } + fflush(file); + check_and_log_file_error(file, "csv"); + return EXIT_SUCCESS; +} + +output_module_t module_csv4rdns_file = { + .name = "csv4rdns", + .init = &csv4rdns_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &csv4rdns_close, + .process_ip = &csv4rdns_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = + "Outputs one or more output fields as a comma-delimited file. By default, the " + "probe module does not filter out duplicates or limit to successful fields, " + "but rather includes all received packets. Fields can be controlled by " + "setting --output-fields. Filtering out failures and duplicate packets can " + "be achieved by setting an --output-filter." +}; diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_csv.h b/rdns_scan/zmap4rdns/src/output_modules/module_csv.h new file mode 100644 index 0000000..582151d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_csv.h @@ -0,0 +1,19 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "../fieldset.h" +#include "output_modules.h" + +int csv_init(struct state_conf *conf, char **fields, int fieldlens); +int csv_process(fieldset_t *fs); +int csv_close(struct state_conf *c, struct state_send *s, struct state_recv *r); + +// output module for rdns scan +int csv4rdns_init(struct state_conf *conf, char **fields, int fieldlens); +int csv4rdns_process(fieldset_t *fs); +int csv4rdns_close(struct state_conf *c, struct state_send *s, struct state_recv *r); diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_json.c b/rdns_scan/zmap4rdns/src/output_modules/module_json.c new file mode 100644 index 0000000..436f3b3 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_json.c @@ -0,0 +1,160 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <assert.h> +#include <errno.h> + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" + +#include <sys/time.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <netinet/udp.h> +#include <netinet/ip_icmp.h> +#include <arpa/inet.h> + +#include <json.h> + +#include "../../lib/logger.h" + +#include "output_modules.h" +#include "../probe_modules/probe_modules.h" + +static FILE *file = NULL; + +int json_output_file_init(struct state_conf *conf, UNUSED const char **fields, + UNUSED int fieldlens) +{ + assert(conf); + if (!conf->output_filename) { + file = stdout; + } else if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal("output-json", + "could not open JSON output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + check_and_log_file_error(file, "json"); + return EXIT_SUCCESS; +} + +char *hex_encode(unsigned char *packet, int buflen) +{ + char *buf = xmalloc(2 * buflen + 1); + for (int i = 0; i < buflen; i++) { + snprintf(buf + (i * 2), 3, "%.2x", packet[i]); + } + buf[buflen * 2] = 0; + return buf; +} + +json_object *fs_to_jsonobj(fieldset_t *fs); +json_object *repeated_to_jsonobj(fieldset_t *fs); + +json_object *field_to_jsonobj(field_t *f) +{ + if (f->type == FS_STRING) { + return json_object_new_string((char *)f->value.ptr); + } else if (f->type == FS_UINT64) { + return json_object_new_int64(f->value.num); + } else if (f->type == FS_BOOL) { + return json_object_new_boolean(f->value.num); + } else if (f->type == FS_BINARY) { + char *encoded = hex_encode(f->value.ptr, f->len); + json_object *t = json_object_new_string(encoded); + free(encoded); + return t; + } else if (f->type == FS_NULL) { + return NULL; + } else if (f->type == FS_FIELDSET) { + return fs_to_jsonobj((fieldset_t *)f->value.ptr); + } else if (f->type == FS_REPEATED) { + return repeated_to_jsonobj((fieldset_t *)f->value.ptr); + } else { + log_fatal("json", "received unknown output type: %i", f->type); + } +} + +json_object *repeated_to_jsonobj(fieldset_t *fs) +{ + json_object *obj = json_object_new_array(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + json_object_array_add(obj, field_to_jsonobj(f)); + } + return obj; +} + +json_object *fs_to_jsonobj(fieldset_t *fs) +{ + json_object *obj = json_object_new_object(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (f->type != FS_NULL) { + json_object_object_add(obj, f->name, field_to_jsonobj(f)); + } + } + return obj; +} + +int json_output_to_file(fieldset_t *fs) +{ + if (!file) { + return EXIT_SUCCESS; + } + json_object *record = fs_to_jsonobj(fs); + fprintf(file, "%s\n", json_object_to_json_string_ext(record, JSON_C_TO_STRING_PLAIN)); + fflush(file); + check_and_log_file_error(file, "json"); + json_object_put(record); + return EXIT_SUCCESS; +} + +int json_output_file_close(UNUSED struct state_conf *c, + UNUSED struct state_send *s, + UNUSED struct state_recv *r) +{ + if (file) { + fflush(file); + fclose(file); + } + return EXIT_SUCCESS; +} + +int print_json_fieldset(fieldset_t *fs) +{ + json_object *record = fs_to_jsonobj(fs); + fprintf(stdout, "%s\n", json_object_to_json_string(record)); + json_object_put(record); + return EXIT_SUCCESS; +} + +output_module_t module_json_file = { + .name = "json", + .init = &json_output_file_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &json_output_file_close, + .process_ip = &json_output_to_file, + .supports_dynamic_output = DYNAMIC_SUPPORT, + .helptext = + "Outputs one or more output fields as a json valid file. By default, the \n" + "probe module does not filter out duplicates or limit to successful fields, \n" + "but rather includes all received packets. Fields can be controlled by \n" + "setting --output-fields. Filtering out failures and duplicate packets can \n" + "be achieved by setting an --output-filter."}; diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_json.h b/rdns_scan/zmap4rdns/src/output_modules/module_json.h new file mode 100644 index 0000000..10526a1 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_json.h @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fieldset.h" + +int print_json_fieldset(fieldset_t *fs); diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_sql.c b/rdns_scan/zmap4rdns/src/output_modules/module_sql.c new file mode 100644 index 0000000..46ea43e --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_sql.c @@ -0,0 +1,168 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <assert.h> +#include <inttypes.h> +#include <mysql/mysql.h> + +#include "../../lib/logger.h" +#include "../fieldset.h" + +#include "output_modules.h" + +#define DEFAULT_ADDR "localhost" +#define DEFAULT_USER "root" +#define DEFAULT_PWD "root" +#define DEFAULT_DATABASE "rdns_scan" +#define DEFAULT_TABLE "zmap_result" + +struct SQL_Info { + char* addr; + char* user; + char* pwd; + char* database; + char* table; +}; + + +static MYSQL* conn; +static struct SQL_Info sql_info; +static int saddr_index = -1; +static int data_index = -1; + + +//sql info format: addr:user:pwd:database:table +int parse_SQL_info(char* str) { + char* p; + int cnt = 0; + p = str; + while(*p != '\0'){ + if(*p == ':') + cnt++; + p++; + } + if(cnt != 4) return 0; + p = str; + sql_info.addr = str; + for (; (*p != ':') && (*p != '\0'); p++); + *p = '\0'; p++; + sql_info.user = p; + for (; (*p != ':') && (*p != '\0'); p++); + *p = '\0'; p++; + sql_info.pwd = p; + for (; (*p != ':') && (*p != '\0'); p++); + *p = '\0'; p++; + sql_info.database = p; + for (; (*p != ':') && (*p != '\0'); p++); + *p = '\0'; p++; + sql_info.table = p; + return 1; +} + +int sql_init(struct state_conf *conf, char **fields, int fieldlens) +{ + assert(conf); + sql_info.addr = DEFAULT_ADDR; + sql_info.user = DEFAULT_USER; + sql_info.pwd = DEFAULT_PWD; + sql_info.database = DEFAULT_DATABASE; + sql_info.table = DEFAULT_TABLE; + + if(conf->output_filename){ + parse_SQL_info(conf->output_filename); + } + if(!(conn = mysql_init(NULL))){ + printf("mysql初始化失败！\n"); + return 1; + } + if(!(conn = mysql_real_connect(conn, sql_info.addr, sql_info.user, sql_info.pwd, sql_info.database, 0, NULL, 0))){ + printf("数据库连接失败！\n"); + return 2; + } + + if (!conf->no_header_row) { + // save the saddr field index and data field index + for (int i = 0; i < fieldlens; i++) { + if(!strcmp(fields[i], "saddr")) + saddr_index = i; + else if(!strcmp(fields[i], "data")){ + data_index = i; + } + } + } + + return EXIT_SUCCESS; +} + +int sql_close(__attribute__((unused)) struct state_conf *c, + __attribute__((unused)) struct state_send *s, + __attribute__((unused)) struct state_recv *r) +{ + mysql_close(conn); + return EXIT_SUCCESS; +} + +int sql_process(fieldset_t *fs) +{ + char cmd[100]; + MYSQL_RES* result; + + if (!conn || data_index == -1 || saddr_index == -1) { + return EXIT_SUCCESS; + } + // if the response is dns response, print the saddr and the + field_t *f = &(fs->fields[data_index]); + unsigned char *buf = (unsigned char *)f->value.ptr; + if(dns_confirm_sql((unsigned char *)f->value.ptr, f->len)){ + f = &(fs->fields[saddr_index]); + + sprintf(cmd, "select * from %s where ip=\"%s\"",sql_info.table,(char*)f->value.ptr); + mysql_query(conn, cmd); + result = mysql_store_result(conn); + if(!mysql_fetch_row(result)){ + sprintf(cmd, "insert into %s (ip,aa,ra,rcode) values (\"%s\",%u,%u,%u)",sql_info.table,(char*)f->value.ptr,((buf[2]&0x04)==0x04),((buf[3]&0x80)==0x80),(buf[3]&0x0f)); + mysql_query(conn, cmd); + } else { + sprintf(cmd, "update %s set aa=%u,ra=%u,rcode=%u where ip=\"%s\"", sql_info.table,((buf[2]&0x04)==0x04),((buf[3]&0x80)==0x80),(buf[3]&0x0f),(char*)f->value.ptr); + mysql_query(conn, cmd); + } + } + return EXIT_SUCCESS; +} + +// confirm the RA bit in the DNS reponse +int dns_confirm_sql(unsigned char *readbuf, size_t len) +{ + if(len < 12) + return 0; + if((readbuf[2]&0x80) == 0x80) + return 1; + else + return 0; +} + +output_module_t module_sql_file = { + .name = "sql", + .init = &sql_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &sql_close, + .process_ip = &sql_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = + "Saving outputs to mysql." +}; diff --git a/rdns_scan/zmap4rdns/src/output_modules/module_sql.h b/rdns_scan/zmap4rdns/src/output_modules/module_sql.h new file mode 100644 index 0000000..c3b7fd3 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/module_sql.h @@ -0,0 +1,7 @@ +#include "../fieldset.h" +#include "output_modules.h" + + +int sql_init(struct state_conf *conf, char **fields, int fieldlens); +int sql_process(fieldset_t *fs); +int sql_close(struct state_conf *c, struct state_send *s, struct state_recv *r);
\ No newline at end of file diff --git a/rdns_scan/zmap4rdns/src/output_modules/output_modules.c b/rdns_scan/zmap4rdns/src/output_modules/output_modules.c new file mode 100644 index 0000000..c445605 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/output_modules.c @@ -0,0 +1,47 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdio.h> +#include <string.h> + +#include "output_modules.h" + +extern output_module_t module_csv_file; +extern output_module_t module_json_file; +extern output_module_t module_csv4rdns_file; +extern output_module_t module_sql_file; + +output_module_t *output_modules[] = { + &module_csv_file, + &module_json_file, + &module_csv4rdns_file, + &moudle_sql_file, + // ADD YOUR MODULE HERE +}; + +output_module_t *get_output_module_by_name(const char *name) +{ + int num_modules = + (int)(sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + if (!strcmp(output_modules[i]->name, name)) { + return output_modules[i]; + } + } + return NULL; +} + +void print_output_modules(void) +{ + int num_modules = + (int)(sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + printf("%s\n", output_modules[i]->name); + } +} + diff --git a/rdns_scan/zmap4rdns/src/output_modules/output_modules.h b/rdns_scan/zmap4rdns/src/output_modules/output_modules.h new file mode 100644 index 0000000..87f24f9 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/output_modules/output_modules.h @@ -0,0 +1,45 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef OUTPUT_MODULES_H +#define OUTPUT_MODULES_H + +#include "../state.h" +#include "../fieldset.h" + +#define NO_DYNAMIC_SUPPORT 0 +#define DYNAMIC_SUPPORT 1 + +// called at scanner initialization +typedef int (*output_init_cb)(struct state_conf *, const char **fields, + int fieldslen); + +// called on packet receipt +typedef int (*output_packet_cb)(fieldset_t *fs); + +// called periodically during the scan +typedef int (*output_update_cb)(struct state_conf *, struct state_send *, + struct state_recv *); + +typedef struct output_module { + const char *name; + int supports_dynamic_output; + unsigned update_interval; + output_init_cb init; + output_update_cb start; + output_update_cb update; + output_update_cb close; + output_packet_cb process_ip; + const char *helptext; +} output_module_t; + +output_module_t *get_output_module_by_name(const char *); + +void print_output_modules(void); + +#endif // HEADER_OUTPUT_MODULES_H diff --git a/rdns_scan/zmap4rdns/src/parser.y b/rdns_scan/zmap4rdns/src/parser.y new file mode 100644 index 0000000..dad3cd7 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/parser.y @@ -0,0 +1,144 @@ +%{ +#include <stdio.h> +#include <string.h> +#include "expression.h" +#include "lexer.h" +#include "filter.h" + +void yyerror(const char *str) +{ + fprintf(stderr,"Parse error: %s\n",str); +} + +int yywrap() +{ + return 1; +} + +extern node_t *zfilter; + +%} + +%union { + int int_literal; + char *string_literal; + struct node_st *expr; +} + +%token '(' ')' T_AND T_OR +%token <int_literal> T_NUMBER +%token <string_literal> T_FIELD +%token T_NOT_EQ T_GT_EQ '>' '<' '=' T_LT_EQ + +%left T_OR +%left T_AND + +%type <expr> filter +%type <expr> number_filter +%type <expr> string_filter +%type <expr> filter_expr + + +%% + +expression: filter_expr + { + zfilter = $1; + } + + +filter_expr: + filter_expr T_OR filter_expr + { + $$ = make_op_node(OR); + $$->left_child = $1; + $$->right_child = $3; + } + | filter_expr T_AND filter_expr + { + $$ = make_op_node(AND); + $$->left_child = $1; + $$->right_child = $3; + } + | '(' filter_expr ')' + { + $$ = $2; + } + | filter + { + $$ = $1; + } + ; + +filter: number_filter + { + $$ = $1; + } + | string_filter + { + $$ = $1; + } + ; + +number_filter: T_FIELD '=' T_NUMBER + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '>' T_NUMBER + { + $$ = make_op_node(GT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '<' T_NUMBER + { + $$ = make_op_node(LT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_NOT_EQ T_NUMBER + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_GT_EQ T_NUMBER + { + $$ = make_op_node(GT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_LT_EQ T_NUMBER + { + $$ = make_op_node(LT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + ; + +string_filter: + T_FIELD '=' T_FIELD + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + | + T_FIELD T_NOT_EQ T_FIELD + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + ; + +%% + + diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.c b/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.c new file mode 100644 index 0000000..70e60de --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.c @@ -0,0 +1,189 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <assert.h> +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> + +#include "../../lib/includes.h" +#include "packet.h" +#include "probe_modules.h" +#include "module_bacnet.h" +#include "module_udp.h" + +#define ICMP_UNREACH_HEADER_SIZE 8 + +#define ZMAP_BACNET_PACKET_LEN (sizeof(struct ether_header) + sizeof(struct ip) + sizeof(struct udphdr) + 0x11) + +probe_module_t module_bacnet; + +static int num_ports; + +static uint8_t bacnet_body[] = {0x0c, 0x02, 0x3f, 0xff, 0xff, 0x19, 0x4b}; +#define BACNET_BODY_LEN 7 + +static inline uint8_t get_invoke_id(uint32_t *validation) +{ + return (uint8_t)((validation[1] >> 24) & 0xFF); +} + +int bacnet_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, void **arg) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + struct bacnet_probe *bnp = (struct bacnet_probe *)&udp_header[1]; + uint8_t *body = (uint8_t *)&bnp[1]; + + make_eth_header(eth_header, src, gw); + + uint16_t ip_len = sizeof(struct ip) + sizeof(struct udphdr) + 0x11; + assert(ip_len <= MAX_PACKET_SIZE); + make_ip_header(ip_header, IPPROTO_UDP, htons(ip_len)); + + uint16_t udp_len = sizeof(struct udphdr) + 0x11; + make_udp_header(udp_header, zconf.target_port, udp_len); + + bnp->vlc.type = ZMAP_BACNET_TYPE_IP; + bnp->vlc.function = ZMAP_BACNET_FUNCTION_UNICAST_NPDU; + bnp->vlc.length = htons(0x11); + + bnp->npdu.version = ZMAP_BACNET_NPDU_VERSION_ASHRAE_135_1995; + bnp->npdu.control = 0x04; + + bnp->apdu.type_flags = 0x00; + bnp->apdu.max_segments_apdu = 0x05; + bnp->apdu.server_choice = 0x0c; + memcpy(body, bacnet_body, BACNET_BODY_LEN); + + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg = aes; + + return EXIT_SUCCESS; +} + +int bacnet_make_packet(void *buf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + struct bacnet_probe *bnp = (struct bacnet_probe *)&udp_header[1]; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + ip_header->ip_sum = 0; + + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + + bnp->apdu.invoke_id = get_invoke_id(validation); + + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ZMAP_BACNET_PACKET_LEN; + + return EXIT_SUCCESS; +} + +int bacnet_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation) +{ + // this will reject packets that aren't UDP or ICMP and fully process ICMP + // packets + if (udp_do_validate_packet(ip_hdr, len, src_ip, validation, num_ports, + zconf.target_port) == PACKET_INVALID) { + return PACKET_INVALID; + } + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + const size_t min_len = + sizeof(struct udphdr) + sizeof(struct bacnet_vlc); + if (udp->uh_ulen < min_len) { + return PACKET_INVALID; + } + struct bacnet_vlc *vlc = + (struct bacnet_vlc *)get_udp_payload(udp, len); + if (vlc->type != ZMAP_BACNET_TYPE_IP) { + return PACKET_INVALID; + } + } + return PACKET_VALID; +} + +void bacnet_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED uint32_t *validation, UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + assert(ip_hdr); + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + assert(udp); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_constchar(fs, "classification", "bacnet"); + fs_add_bool(fs, "success", 1); + fs_add_null_icmp(fs); + uint32_t udp_offset = sizeof(struct ether_header) + ip_hdr->ip_hl * 4; + uint32_t payload_offset = udp_offset + sizeof(struct udphdr); + assert(payload_offset < len); + uint8_t *payload = get_udp_payload(udp, len); + uint32_t payload_len = len - payload_offset; + fs_add_binary(fs, "udp_payload", payload_len, (void *)payload, 0); + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "udp_payload"); + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +int bacnet_global_initialize(struct state_conf *conf) +{ + num_ports = conf->source_port_last - conf->source_port_first + 1; + return EXIT_SUCCESS; +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "udp_payload", .type = "binary", .desc = "UDP payload"}, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_bacnet = { + .name = "bacnet", + .max_packet_length = ZMAP_BACNET_PACKET_LEN, + .pcap_filter = "udp || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .thread_initialize = &bacnet_init_perthread, + .global_initialize = &bacnet_global_initialize, + .make_packet = &bacnet_make_packet, + .print_packet = &udp_print_packet, + .validate_packet = &bacnet_validate_packet, + .process_packet = &bacnet_process_packet, + .close = &udp_global_cleanup, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.h b/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.h new file mode 100644 index 0000000..6dc4091 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_bacnet.h @@ -0,0 +1,57 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_MODULE_BACNET_H +#define ZMAP_MODULE_BACNET_H + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> + +#include "probe_modules.h" + +extern probe_module_t module_bacnet; + +struct __attribute__((__packed__)) bacnet_vlc { + uint8_t type; + uint8_t function; + uint16_t length; +}; +typedef struct bacnet_vlc bacnet_vlc_t; + +struct __attribute__((__packed__)) bacnet_npdu { + uint8_t version; + uint8_t control; +}; +typedef struct bacnet_npdu bacnet_npdu_t; + +struct __attribute__((__packed__)) bacnet_apdu { + uint8_t type_flags; + uint8_t max_segments_apdu; + uint8_t invoke_id; + uint8_t server_choice; +}; +typedef struct bacnet_apdu bacnet_apdu_t; + +struct __attribute__((__packed__)) bacnet_probe { + struct bacnet_vlc vlc; + struct bacnet_npdu npdu; + struct bacnet_apdu apdu; +}; +typedef struct bacnet_probe bacnet_probe_t; + +typedef struct bacnet_oid bacnet_oid_t; + +#define ZMAP_BACNET_TYPE_IP 0x81 +#define ZMAP_BACNET_FUNCTION_UNICAST_NPDU 0x0a +#define ZMAP_BACNET_NPDU_VERSION_ASHRAE_135_1995 0x01 +#define ZMAP_BACNET_SERVER_CHOICE_READ_PROPERTY 0x0c + +#endif /* ZMAP_MODULE_BACNET_H */ diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_dns.c b/rdns_scan/zmap4rdns/src/probe_modules/module_dns.c new file mode 100644 index 0000000..0076162 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_dns.c @@ -0,0 +1,1094 @@ +/* + * ZMap Copyright 2015 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// Module for scanning for open UDP DNS resolvers. +// +// This module optionally takes in an argument of the form "TYPE,QUESTION" +// (e.g. "A,google.com"). +// +// Given no arguments it will default to asking for an A record for +// www.google.com. +// +// This module does minimal answer verification. It only verifies that the +// response roughly looks like a DNS response. It will not, for example, +// require the QR bit be set to 1. All such analysis should happen offline. +// Specifically, to be included in the output it requires: +// - That the response packet is >= the query packet. +// - That the ports match and the packet is complete. +// To be marked as success it also requires: +// - That the response bytes that should be the ID field matches the send bytes. +// - That the response bytes that should be question match send bytes. +// To be marked as app_success it also requires: +// - That the QR bit be 1 and rcode == 0. +// +// Usage: +// zmap -p 53 --probe-module=dns --probe-args="ANY,www.example.com" +// -O json --output-fields=* 8.8.8.8 +// +// We also support multiple questions, of the form: +// "A,example.com;AAAA,www.example.com" This requires --probes=X, where X +// matches the number of questions in --probe-args, and either +// --output-filter="" or --output-module=csv to remove the implicit +// "filter_duplicates" configuration flag. +// + +#include "module_dns.h" +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../../lib/random.h" +#include "../../lib/xalloc.h" +#include "probe_modules.h" +#include "packet.h" +#include "logger.h" +#include "module_udp.h" +#include "../fieldset.h" + +#define DNS_PAYLOAD_LEN_LIMIT 512 // This is arbitrary +#define UDP_HEADER_LEN 8 +#define PCAP_SNAPLEN 1500 // This is even more arbitrary +#define MAX_QTYPE 255 +#define ICMP_UNREACH_HEADER_SIZE 8 +#define BAD_QTYPE_STR "BAD QTYPE" +#define BAD_QTYPE_VAL -1 +#define MAX_LABEL_RECURSION 10 +#define DNS_QR_ANSWER 1 + +// Note: each label has a max length of 63 bytes. So someone has to be doing +// something really annoying. Will raise a warning. +// THIS INCLUDES THE NULL BYTE +#define MAX_NAME_LENGTH 512 + +#if defined(__NetBSD__) && !defined(__cplusplus) && defined(bool) +#undef bool +#endif + +typedef uint8_t bool; + +// zmap boilerplate +probe_module_t module_dns; +static int num_ports; + +const char default_domain[] = "www.google.com"; +const uint16_t default_qtype = DNS_QTYPE_A; + +static char **dns_packets; +static uint16_t *dns_packet_lens; // Not including udp header +static uint16_t *qname_lens; +static char **qnames; +static uint16_t *qtypes; +static int num_questions = 0; + +/* Array of qtypes we support. Jumping through some hoops (1 level of + * indirection) so the per-packet processing time is fast. Keep this in sync + * with: dns_qtype (.h) qtype_strid_to_qtype (below) qtype_qtype_to_strid + * (below, and setup_qtype_str_map()) + */ +const char *qtype_strs[] = {"A", "NS", "CNAME", "SOA", "PTR", + "MX", "TXT", "AAAA", "RRSIG", "ALL"}; +const int qtype_strs_len = 10; + +const dns_qtype qtype_strid_to_qtype[] = { + DNS_QTYPE_A, DNS_QTYPE_NS, DNS_QTYPE_CNAME, DNS_QTYPE_SOA, + DNS_QTYPE_PTR, DNS_QTYPE_MX, DNS_QTYPE_TXT, DNS_QTYPE_AAAA, + DNS_QTYPE_RRSIG, DNS_QTYPE_ALL}; + +int8_t qtype_qtype_to_strid[256] = {BAD_QTYPE_VAL}; + +void setup_qtype_str_map() +{ + qtype_qtype_to_strid[DNS_QTYPE_A] = 0; + qtype_qtype_to_strid[DNS_QTYPE_NS] = 1; + qtype_qtype_to_strid[DNS_QTYPE_CNAME] = 2; + qtype_qtype_to_strid[DNS_QTYPE_SOA] = 3; + qtype_qtype_to_strid[DNS_QTYPE_PTR] = 4; + qtype_qtype_to_strid[DNS_QTYPE_MX] = 5; + qtype_qtype_to_strid[DNS_QTYPE_TXT] = 6; + qtype_qtype_to_strid[DNS_QTYPE_AAAA] = 7; + qtype_qtype_to_strid[DNS_QTYPE_RRSIG] = 8; + qtype_qtype_to_strid[DNS_QTYPE_ALL] = 9; +} + +static uint16_t qtype_str_to_code(const char *str) +{ + for (int i = 0; i < qtype_strs_len; i++) { + if (strcmp(qtype_strs[i], str) == 0) + return qtype_strid_to_qtype[i]; + } + return 0; +} + +static uint16_t domain_to_qname(char **qname_handle, const char *domain) +{ + // String + 1byte header + null byte + uint16_t len = strlen(domain) + 1 + 1; + char *qname = xmalloc(len); + // Add a . before the domain. This will make the following simpler. + qname[0] = '.'; + // Move the domain into the qname buffer. + strcpy(qname + 1, domain); + for (int i = 0; i < len; i++) { + if (qname[i] == '.') { + int j; + for (j = i + 1; j < (len - 1); j++) { + if (qname[j] == '.') { + break; + } + } + qname[i] = j - i - 1; + } + } + *qname_handle = qname; + assert((*qname_handle)[len - 1] == '\0'); + return len; +} + +static int build_global_dns_packets(char *domains[], int num_domains, size_t *max_len) +{ + size_t _max_len = 0; + for (int i = 0; i < num_domains; i++) { + + qname_lens[i] = domain_to_qname(&qnames[i], domains[i]); + if (domains[i] != (char *)default_domain) { + free(domains[i]); + } + uint16_t len = sizeof(dns_header) + qname_lens[i] + + sizeof(dns_question_tail); + dns_packet_lens[i] = len; + if (len > _max_len) { + _max_len = len; + } + + if (dns_packet_lens[i] > DNS_PAYLOAD_LEN_LIMIT) { + log_fatal("dns", + "DNS packet bigger (%d) than our limit (%d)", + dns_packet_lens[i], DNS_PAYLOAD_LEN_LIMIT); + return EXIT_FAILURE; + } + + dns_packets[i] = xmalloc(dns_packet_lens[i]); + + dns_header *dns_header_p = (dns_header *)dns_packets[i]; + char *qname_p = dns_packets[i] + sizeof(dns_header); + dns_question_tail *tail_p = + (dns_question_tail *)(dns_packets[i] + sizeof(dns_header) + + qname_lens[i]); + + // All other header fields should be 0. Except id, which we set + // per thread. Please recurse as needed. + dns_header_p->rd = 1; // Is one bit. Don't need htons + // We have 1 question + dns_header_p->qdcount = htons(1); + memcpy(qname_p, qnames[i], qname_lens[i]); + // Set the qtype to what we passed from args + tail_p->qtype = htons(qtypes[i]); + // Set the qclass to The Internet + tail_p->qclass = htons(0x01); + } + *max_len = _max_len; + return EXIT_SUCCESS; +} + +static uint16_t get_name_helper(const char *data, uint16_t data_len, + const char *payload, uint16_t payload_len, + char *name, uint16_t name_len, + uint16_t recursion_level) +{ + log_trace("dns", + "_get_name_helper IN, datalen: %d namelen: %d recursion: %d", + data_len, name_len, recursion_level); + if (data_len == 0 || name_len == 0 || payload_len == 0) { + log_trace( + "dns", + "_get_name_helper OUT, err. 0 length field. datalen %d namelen %d payloadlen %d", + data_len, name_len, payload_len); + return 0; + } + if (recursion_level > MAX_LABEL_RECURSION) { + log_trace("dns", "_get_name_helper OUT. ERR, MAX RECURSION"); + return 0; + } + uint16_t bytes_consumed = 0; + // The start of data is either a sequence of labels or a ptr. + while (data_len > 0) { + uint8_t byte = data[0]; + // Is this a pointer? + if (byte >= 0xc0) { + log_trace("dns", "_get_name_helper, ptr encountered"); + // Do we have enough bytes to check ahead? + if (data_len < 2) { + log_trace( + "dns", + "_get_name_helper OUT. ptr byte encountered. No offset. ERR."); + return 0; + } + // No. ntohs isn't needed here. It's because of + // the upper 2 bits indicating a pointer. + uint16_t offset = + ((byte & 0x03) << 8) | (uint8_t)data[1]; + log_trace("dns", "_get_name_helper. ptr offset 0x%x", + offset); + if (offset >= payload_len) { + log_trace( + "dns", + "_get_name_helper OUT. offset exceeded payload len %d ERR", + payload_len); + return 0; + } + + // We need to add a dot if we are: + // -- Not first level recursion. + // -- have consumed bytes + if (recursion_level > 0 || bytes_consumed > 0) { + + if (name_len < 1) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + name[0] = '.'; + name++; + name_len--; + } + uint16_t rec_bytes_consumed = get_name_helper( + payload + offset, payload_len - offset, payload, + payload_len, name, name_len, recursion_level + 1); + // We are done so don't bother to increment the + // pointers. + if (rec_bytes_consumed == 0) { + log_trace( + "dns", + "_get_name_helper OUT. rec level %d failed", + recursion_level); + return 0; + } else { + bytes_consumed += 2; + log_trace( + "dns", + "_get_name_helper OUT. rec level %d success. " + "%d rec bytes consumed. %d bytes consumed.", + recursion_level, rec_bytes_consumed, + bytes_consumed); + return bytes_consumed; + } + } else if (byte == '\0') { + // don't bother with pointer incrementation. We're done. + bytes_consumed += 1; + log_trace( + "dns", + "_get_name_helper OUT. rec level %d success. %d bytes consumed.", + recursion_level, bytes_consumed); + return bytes_consumed; + } else { + log_trace("dns", + "_get_name_helper, segment 0x%hx encountered", + byte); + // We've now consumed a byte. + ++data; + --data_len; + // Mark byte consumed after we check for first + // iteration. Do we have enough data left (must have + // null byte too)? + if ((byte + 1) > data_len) { + log_trace( + "dns", + "_get_name_helper OUT. ERR. Not enough data for segment %hd"); + return 0; + } + // If we've consumed any bytes and are in a label, we're + // in a label chain. We need to add a dot. + if (bytes_consumed > 0) { + + if (name_len < 1) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + name[0] = '.'; + name++; + name_len--; + } + // Now we've consumed a byte. + ++bytes_consumed; + // Did we run out of our arbitrary buffer? + if (byte > name_len) { + log_warn( + "dns", + "Exceeded static name field allocation."); + return 0; + } + + assert(data_len > 0); + memcpy(name, data, byte); + name += byte; + name_len -= byte; + data_len -= byte; + data += byte; + bytes_consumed += byte; + // Handled in the byte+1 check above. + assert(data_len > 0); + } + } + // We should never get here. + // For each byte we either have: + // -- a ptr, which terminates + // -- a null byte, which terminates + // -- a segment length which either terminates or ensures we keep + // looping + assert(0); + return 0; +} + +// data: Where we are in the dns payload +// payload: the entire udp payload +static char *get_name(const char *data, uint16_t data_len, const char *payload, + uint16_t payload_len, uint16_t *bytes_consumed) +{ + log_trace("dns", "call to get_name, data_len: %d", data_len); + char *name = xmalloc(MAX_NAME_LENGTH); + *bytes_consumed = get_name_helper(data, data_len, payload, payload_len, + name, MAX_NAME_LENGTH - 1, 0); + if (*bytes_consumed == 0) { + free(name); + return NULL; + } + // Our memset ensured null byte. + assert(name[MAX_NAME_LENGTH - 1] == '\0'); + log_trace( + "dns", + "return success from get_name, bytes_consumed: %d, string: %s", + *bytes_consumed, name); + return name; +} + +static bool process_response_question(char **data, uint16_t *data_len, + const char *payload, uint16_t payload_len, + fieldset_t *list) +{ + // Payload is the start of the DNS packet, including header + // data is handle to the start of this RR + // data_len is a pointer to the how much total data we have to work + // with. This is awful. I'm bad and should feel bad. + uint16_t bytes_consumed = 0; + char *question_name = + get_name(*data, *data_len, payload, payload_len, &bytes_consumed); + // Error. + if (question_name == NULL) { + return 1; + } + assert(bytes_consumed > 0); + if ((bytes_consumed + sizeof(dns_question_tail)) > *data_len) { + free(question_name); + return 1; + } + dns_question_tail *tail = (dns_question_tail *)(*data + bytes_consumed); + uint16_t qtype = ntohs(tail->qtype); + uint16_t qclass = ntohs(tail->qclass); + // Build our new question fieldset + fieldset_t *qfs = fs_new_fieldset(NULL); + fs_add_unsafe_string(qfs, "name", question_name, 1); + fs_add_uint64(qfs, "qtype", qtype); + if (qtype > MAX_QTYPE || qtype_qtype_to_strid[qtype] == BAD_QTYPE_VAL) { + fs_add_string(qfs, "qtype_str", (char *)BAD_QTYPE_STR, 0); + } else { + // I've written worse things than this 3rd arg. But I want to be + // fast. + fs_add_string(qfs, "qtype_str", + (char *)qtype_strs[qtype_qtype_to_strid[qtype]], + 0); + } + fs_add_uint64(qfs, "qclass", qclass); + // Now we're adding the new fs to the list. + fs_add_fieldset(list, NULL, qfs); + // Now update the pointers. + *data = *data + bytes_consumed + sizeof(dns_question_tail); + *data_len = *data_len - bytes_consumed - sizeof(dns_question_tail); + return 0; +} + +static bool process_response_answer(char **data, uint16_t *data_len, + const char *payload, uint16_t payload_len, + fieldset_t *list) +{ + log_trace("dns", "call to process_response_answer, data_len: %d", + *data_len); + // Payload is the start of the DNS packet, including header + // data is handle to the start of this RR + // data_len is a pointer to the how much total data we have to work + // with. This is awful. I'm bad and should feel bad. + uint16_t bytes_consumed = 0; + char *answer_name = + get_name(*data, *data_len, payload, payload_len, &bytes_consumed); + // Error. + if (answer_name == NULL) { + return 1; + } + assert(bytes_consumed > 0); + if ((bytes_consumed + sizeof(dns_answer_tail)) > *data_len) { + free(answer_name); + return 1; + } + dns_answer_tail *tail = (dns_answer_tail *)(*data + bytes_consumed); + uint16_t type = ntohs(tail->type); + uint16_t class = ntohs(tail->class); + uint32_t ttl = ntohl(tail->ttl); + uint16_t rdlength = ntohs(tail->rdlength); + char *rdata = tail->rdata; + + if ((rdlength + bytes_consumed + sizeof(dns_answer_tail)) > *data_len) { + free(answer_name); + return 1; + } + // Build our new question fieldset + fieldset_t *afs = fs_new_fieldset(NULL); + fs_add_unsafe_string(afs, "name", answer_name, 1); + fs_add_uint64(afs, "type", type); + if (type > MAX_QTYPE || qtype_qtype_to_strid[type] == BAD_QTYPE_VAL) { + fs_add_string(afs, "type_str", (char *)BAD_QTYPE_STR, 0); + } else { + // I've written worse things than this 3rd arg. But I want to be + // fast. + fs_add_string(afs, "type_str", + (char *)qtype_strs[qtype_qtype_to_strid[type]], + 0); + } + fs_add_uint64(afs, "class", class); + fs_add_uint64(afs, "ttl", ttl); + fs_add_uint64(afs, "rdlength", rdlength); + + // XXX Fill this out for the other types we care about. + if (type == DNS_QTYPE_NS || type == DNS_QTYPE_CNAME) { + uint16_t rdata_bytes_consumed = 0; + char *rdata_name = get_name(rdata, rdlength, payload, + payload_len, &rdata_bytes_consumed); + if (rdata_name == NULL) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + fs_add_unsafe_string(afs, "rdata", rdata_name, 1); + } + } else if (type == DNS_QTYPE_MX) { + uint16_t rdata_bytes_consumed = 0; + if (rdlength <= 4) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + char *rdata_name = + get_name(rdata + 2, rdlength - 2, payload, + payload_len, &rdata_bytes_consumed); + if (rdata_name == NULL) { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + // (largest value 16bit) + " " + answer + null + char *rdata_with_pref = + xmalloc(5 + 1 + strlen(rdata_name) + 1); + + uint8_t num_printed = + snprintf(rdata_with_pref, 6, "%hu ", + ntohs(*(uint16_t *)rdata)); + memcpy(rdata_with_pref + num_printed, + rdata_name, strlen(rdata_name)); + fs_add_uint64(afs, "rdata_is_parsed", 1); + fs_add_unsafe_string(afs, "rdata", + rdata_with_pref, 1); + } + } + } else if (type == DNS_QTYPE_TXT) { + if (rdlength >= 1 && (rdlength - 1) != *(uint8_t *)rdata) { + log_warn( + "dns", + "TXT record with wrong TXT len. Not processing."); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *txt = xmalloc(rdlength); + memcpy(txt, rdata + 1, rdlength - 1); + fs_add_unsafe_string(afs, "rdata", txt, 1); + } + } else if (type == DNS_QTYPE_A) { + if (rdlength != 4) { + log_warn( + "dns", + "A record with IP of length %d. Not processing.", + rdlength); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *addr = + strdup(inet_ntoa(*(struct in_addr *)rdata)); + fs_add_unsafe_string(afs, "rdata", addr, 1); + } + } else if (type == DNS_QTYPE_AAAA) { + if (rdlength != 16) { + log_warn( + "dns", + "AAAA record with IP of length %d. Not processing.", + rdlength); + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } else { + fs_add_uint64(afs, "rdata_is_parsed", 1); + char *ipv6_str = xmalloc(INET6_ADDRSTRLEN); + + inet_ntop(AF_INET6, (struct sockaddr_in6 *)rdata, + ipv6_str, INET6_ADDRSTRLEN); + + fs_add_unsafe_string(afs, "rdata", ipv6_str, 1); + } + } else { + fs_add_uint64(afs, "rdata_is_parsed", 0); + fs_add_binary(afs, "rdata", rdlength, rdata, 0); + } + // Now we're adding the new fs to the list. + fs_add_fieldset(list, NULL, afs); + // Now update the pointers. + *data = *data + bytes_consumed + sizeof(dns_answer_tail) + rdlength; + *data_len = + *data_len - bytes_consumed - sizeof(dns_answer_tail) - rdlength; + log_trace("dns", + "return success from process_response_answer, data_len: %d", + *data_len); + return 0; +} + +/* + * Start of required zmap exports. + */ + +static int dns_global_initialize(struct state_conf *conf) +{ + num_questions = conf->packet_streams; + if (num_questions < 1) { + log_fatal("dns", + "Invalid number of probes for the DNS module: %i", + num_questions); + } + + // Setup the global structures + dns_packets = xmalloc(sizeof(char *) * num_questions); + dns_packet_lens = xmalloc(sizeof(uint16_t) * num_questions); + qname_lens = xmalloc(sizeof(uint16_t) * num_questions); + qnames = xmalloc(sizeof(char *) * num_questions); + qtypes = xmalloc(sizeof(uint16_t) * num_questions); + + char *qtype_str = NULL; + char **domains = (char **)xmalloc(sizeof(char *) * num_questions); + + for (int i = 0; i < num_questions; i++) { + domains[i] = (char *)default_domain; + qtypes[i] = default_qtype; + } + + num_ports = conf->source_port_last - conf->source_port_first + 1; + + setup_qtype_str_map(); + + if (conf->probe_args) { + int arg_strlen = strlen(conf->probe_args); + char *arg_pos = conf->probe_args; + + for (int i = 0; i < num_questions; i++) { + if (arg_pos >= (conf->probe_args + arg_strlen)) { + log_fatal( + "dns", + "More probes than questions configured. Add additional questions."); + } + + char *probe_q_delimiter_p = strchr(arg_pos, ','); + char *probe_arg_delimiter_p = strchr(arg_pos, ';'); + + if (probe_q_delimiter_p == NULL || + probe_q_delimiter_p == arg_pos || + arg_pos + strlen(arg_pos) == + (probe_q_delimiter_p + 1) || + (probe_arg_delimiter_p == NULL && + (i + 1) != num_questions)) { + log_fatal( + "dns", + "Invalid probe args. Format: \"A,google.com\" or \"A,google.com;A,example.com\""); + } + + int domain_len = 0; + + if (probe_arg_delimiter_p) { + domain_len = probe_arg_delimiter_p - + probe_q_delimiter_p - 1; + } else { + domain_len = strlen(probe_q_delimiter_p); + } + assert(domain_len > 0); + + domains[i] = xmalloc(domain_len + 1); + strncpy(domains[i], probe_q_delimiter_p + 1, + domain_len); + domains[i][domain_len] = '\0'; + + qtype_str = xmalloc(probe_q_delimiter_p - arg_pos + 1); + strncpy(qtype_str, arg_pos, + probe_q_delimiter_p - arg_pos); + qtype_str[probe_q_delimiter_p - arg_pos] = '\0'; + + qtypes[i] = qtype_str_to_code(qtype_str); + free(qtype_str); + if (!qtypes[i]) { + log_fatal("dns", "Incorrect qtype supplied. %s", + qtype_str); + return EXIT_FAILURE; + } + + arg_pos = probe_q_delimiter_p + domain_len + 2; + } + + if (arg_pos != conf->probe_args + arg_strlen + 2) { + log_fatal( + "dns", + "More args than probes passed. Add additional probes."); + } + } + size_t max_payload_len; + int ret = build_global_dns_packets(domains, num_questions, &max_payload_len); + module_dns.max_packet_length = max_payload_len + + sizeof(struct ether_header) + + sizeof(struct ip) + + sizeof(struct udphdr); + return ret; +} + +static int dns_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv) +{ + if (dns_packets) { + for (int i = 0; i < num_questions; i++) { + if (dns_packets[i]) { + free(dns_packets[i]); + } + } + free(dns_packets); + } + dns_packets = NULL; + + if (qnames) { + for (int i = 0; i < num_questions; i++) { + if (qnames[i]) { + free(qnames[i]); + } + } + free(qnames); + } + qnames = NULL; + + if (dns_packet_lens) { + free(dns_packet_lens); + } + + if (qname_lens) { + free(qname_lens); + } + + if (qtypes) { + free(qtypes); + } + + return EXIT_SUCCESS; +} + +int dns_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + // Setup assuming num_questions == 0 + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + dns_packet_lens[0]); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + len = sizeof(struct udphdr) + dns_packet_lens[0]; + make_udp_header(udp_header, zconf.target_port, len); + + char *payload = (char *)(&udp_header[1]); + + memcpy(payload, dns_packets[0], dns_packet_lens[0]); + + return EXIT_SUCCESS; +} + +int dns_make_packet(void *buf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + + // For num_questions == 1, we handle this in per-thread init. Do less + // work + if (num_questions > 1) { + + uint16_t encoded_len = + htons(sizeof(struct ip) + sizeof(struct udphdr) + + dns_packet_lens[probe_num]); + make_ip_header(ip_header, IPPROTO_UDP, encoded_len); + + encoded_len = + sizeof(struct udphdr) + dns_packet_lens[probe_num]; + make_udp_header(udp_header, ntohs(udp_header->uh_dport), + encoded_len); + + char *payload = (char *)(&udp_header[1]); + *buf_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + dns_packet_lens[probe_num]; + + assert(*buf_len <= MAX_PACKET_SIZE); + + memcpy(payload, dns_packets[probe_num], + dns_packet_lens[probe_num]); + } + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + + dns_header *dns_header_p = (dns_header *)&udp_header[1]; + + dns_header_p->id = validation[2] & 0xFFFF; + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + return EXIT_SUCCESS; +} + +void dns_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(&iph[1]); + fprintf(fp, PRINT_PACKET_SEP); + fprintf(fp, "dns { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +int dns_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation) +{ + // this does the heavy lifting including ICMP validation + if (udp_do_validate_packet(ip_hdr, len, src_ip, validation, num_ports, + zconf.target_port) == PACKET_INVALID) { + return PACKET_INVALID; + } + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + // verify our packet length + uint16_t udp_len = ntohs(udp->uh_ulen); + int match = 0; + for (int i = 0; i < num_questions; i++) { + if (udp_len >= dns_packet_lens[i]) { + match += 1; + } + } + if (match == 0) { + return PACKET_INVALID; + } + if (len < udp_len) { + return PACKET_INVALID; + } + } + return PACKET_VALID; +} + +void dns_add_null_fs(fieldset_t *fs) { + fs_add_null(fs, "dns_id"); + fs_add_null(fs, "dns_rd"); + fs_add_null(fs, "dns_tc"); + fs_add_null(fs, "dns_aa"); + fs_add_null(fs, "dns_opcode"); + fs_add_null(fs, "dns_qr"); + fs_add_null(fs, "dns_rcode"); + fs_add_null(fs, "dns_cd"); + fs_add_null(fs, "dns_ad"); + fs_add_null(fs, "dns_z"); + fs_add_null(fs, "dns_ra"); + fs_add_null(fs, "dns_qdcount"); + fs_add_null(fs, "dns_ancount"); + fs_add_null(fs, "dns_nscount"); + fs_add_null(fs, "dns_arcount"); + + fs_add_repeated(fs, "dns_questions", + fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_answers", fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_authorities", + fs_new_repeated_fieldset()); + fs_add_repeated(fs, "dns_additionals", + fs_new_repeated_fieldset()); + + fs_add_uint64(fs, "dns_parse_err", 1); + fs_add_uint64(fs, "dns_unconsumed_bytes", 0); +} + +void dns_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + uint32_t *validation, + __attribute__((unused)) struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp_hdr = get_udp_header(ip_hdr, len); + assert(udp_hdr); + uint16_t udp_len = ntohs(udp_hdr->uh_ulen); + + int match = 0; + bool is_valid = 0; + for (int i = 0; i < num_questions; i++) { + if (udp_len < dns_packet_lens[i]) { + continue; + } + match += 1; + + char *qname_p = NULL; + dns_question_tail *tail_p = NULL; + dns_header *dns_header_p = (dns_header *)&udp_hdr[1]; + // verify our dns transaction id + if (dns_header_p->id == (validation[2] & 0xFFFF)) { + // Verify our question + qname_p = + (char *)dns_header_p + sizeof(dns_header); + tail_p = + (dns_question_tail *)(dns_packets[i] + + sizeof(dns_header) + + qname_lens[i]); + // Verify our qname + if (strcmp(qnames[i], qname_p) == 0) { + // Verify the qtype and qclass. + if (tail_p->qtype == htons(qtypes[i]) && + tail_p->qclass == htons(0x01)) { + is_valid = 1; + break; + } + } + } + } + assert(match > 0); + + dns_header *dns_hdr = (dns_header *)&udp_hdr[1]; + uint16_t qr = dns_hdr->qr; + uint16_t rcode = dns_hdr->rcode; + // Success: Has the right validation bits and the right Q + // App success: has qr and rcode bits right + // Any app level parsing issues: dns_parse_err + // + fs_add_uint64(fs, "sport", ntohs(udp_hdr->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp_hdr->uh_dport)); + + // High level info + fs_add_string(fs, "classification", (char *)"dns", 0); + fs_add_bool(fs, "success", is_valid); + // additional UDP information + fs_add_bool(fs, "app_success", + is_valid && (qr == DNS_QR_ANSWER) && + (rcode == DNS_RCODE_NOERR)); + // ICMP info + fs_add_null_icmp(fs); + fs_add_uint64(fs, "udp_len", udp_len); + // DNS data + if (!is_valid) { + dns_add_null_fs(fs); + } else { + // DNS header + fs_add_uint64(fs, "dns_id", ntohs(dns_hdr->id)); + fs_add_uint64(fs, "dns_rd", dns_hdr->rd); + fs_add_uint64(fs, "dns_tc", dns_hdr->tc); + fs_add_uint64(fs, "dns_aa", dns_hdr->aa); + fs_add_uint64(fs, "dns_opcode", dns_hdr->opcode); + fs_add_uint64(fs, "dns_qr", qr); + fs_add_uint64(fs, "dns_rcode", rcode); + fs_add_uint64(fs, "dns_cd", dns_hdr->cd); + fs_add_uint64(fs, "dns_ad", dns_hdr->ad); + fs_add_uint64(fs, "dns_z", dns_hdr->z); + fs_add_uint64(fs, "dns_ra", dns_hdr->ra); + fs_add_uint64(fs, "dns_qdcount", + ntohs(dns_hdr->qdcount)); + fs_add_uint64(fs, "dns_ancount", + ntohs(dns_hdr->ancount)); + fs_add_uint64(fs, "dns_nscount", + ntohs(dns_hdr->nscount)); + fs_add_uint64(fs, "dns_arcount", + ntohs(dns_hdr->arcount)); + // And now for the complicated part. Hierarchical data. + char *data = ((char *)dns_hdr) + sizeof(dns_header); + uint16_t data_len = + udp_len - sizeof(udp_hdr) - sizeof(dns_header); + bool err = 0; + // Questions + fieldset_t *list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->qdcount) && !err; + i++) { + err = process_response_question( + &data, &data_len, (char *)dns_hdr, udp_len, + list); + } + fs_add_repeated(fs, "dns_questions", list); + // Answers + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->ancount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_answers", list); + // Authorities + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->nscount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_authorities", list); + // Additionals + list = fs_new_repeated_fieldset(); + for (int i = 0; i < ntohs(dns_hdr->arcount) && !err; + i++) { + err = process_response_answer(&data, &data_len, + (char *)dns_hdr, + udp_len, list); + } + fs_add_repeated(fs, "dns_additionals", list); + // Do we have unconsumed data? + if (data_len != 0) { + err = 1; + } + // Did we parse OK? + fs_add_uint64(fs, "dns_parse_err", err); + fs_add_uint64(fs, "dns_unconsumed_bytes", data_len); + } + // Now the raw stuff. + fs_add_binary(fs, "raw_data", (udp_len - sizeof(struct udphdr)), + (void *)&udp_hdr[1], 0); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_bool(fs, "app_success", 0); + // Populate all ICMP Fields + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + fs_add_null(fs, "udp_len"); + dns_add_null_fs(fs); + fs_add_binary(fs, "raw_data", len, (char *)packet, 0); + } else { + // This should not happen. Both the pcap filter and validate + // packet prevent this. + log_fatal("dns", "Die. This can only happen if you " + "change the pcap filter and don't update the " + "process function."); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "app_success", + .type = "bool", + .desc = "Is the RA bit set with no error code?"}, + ICMP_FIELDSET_FIELDS, + {.name = "udp_len", .type = "int", .desc = "UDP packet length"}, + {.name = "dns_id", .type = "int", .desc = "DNS transaction ID"}, + {.name = "dns_rd", .type = "int", .desc = "DNS recursion desired"}, + {.name = "dns_tc", .type = "int", .desc = "DNS packet truncated"}, + {.name = "dns_aa", .type = "int", .desc = "DNS authoritative answer"}, + {.name = "dns_opcode", .type = "int", .desc = "DNS opcode (query type)"}, + {.name = "dns_qr", .type = "int", .desc = "DNS query(0) or response (1)"}, + {.name = "dns_rcode", .type = "int", .desc = "DNS response code"}, + {.name = "dns_cd", .type = "int", .desc = "DNS checking disabled"}, + {.name = "dns_ad", .type = "int", .desc = "DNS authenticated data"}, + {.name = "dns_z", .type = "int", .desc = "DNS reserved"}, + {.name = "dns_ra", .type = "int", .desc = "DNS recursion available"}, + {.name = "dns_qdcount", .type = "int", .desc = "DNS number questions"}, + {.name = "dns_ancount", .type = "int", .desc = "DNS number answer RR's"}, + {.name = "dns_nscount", + .type = "int", + .desc = "DNS number NS RR's in authority section"}, + {.name = "dns_arcount", + .type = "int", + .desc = "DNS number additional RR's"}, + {.name = "dns_questions", .type = "repeated", .desc = "DNS question list"}, + {.name = "dns_answers", .type = "repeated", .desc = "DNS answer list"}, + {.name = "dns_authorities", + .type = "repeated", + .desc = "DNS authority list"}, + {.name = "dns_additionals", + .type = "repeated", + .desc = "DNS additional list"}, + {.name = "dns_parse_err", + .type = "int", + .desc = "Problem parsing the DNS response"}, + {.name = "dns_unconsumed_bytes", + .type = "int", + .desc = "Bytes left over when parsing" + " the DNS response"}, + {.name = "raw_data", .type = "binary", .desc = "UDP payload"}, +}; + +probe_module_t module_dns = { + .name = "dns", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = PCAP_SNAPLEN, + .port_args = 1, + .thread_initialize = &dns_init_perthread, + .global_initialize = &dns_global_initialize, + .make_packet = &dns_make_packet, + .print_packet = &dns_print_packet, + .validate_packet = &dns_validate_packet, + .process_packet = &dns_process_packet, + .close = &dns_global_cleanup, + .output_type = OUTPUT_TYPE_DYNAMIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = + "This module sends out DNS queries and parses basic responses. " + "By default, the module will perform an A record lookup for " + "google.com. You can specify other queries using the --probe-args " + "argument in the form: 'type,query', e.g. 'A,google.com'. The module " + "supports sending the the following types: of queries: A, NS, CNAME, SOA, " + "PTR, MX, TXT, AAAA, RRSIG, and ALL. The module will accept and attempt " + "to parse all DNS responses. There is currently support for parsing out " + "full data from A, NS, CNAME, MX, TXT, and AAAA. Any other types will be " + "output in raw form." + +}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_dns.h b/rdns_scan/zmap4rdns/src/probe_modules/module_dns.h new file mode 100644 index 0000000..5f0f5e1 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_dns.h @@ -0,0 +1,69 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> + +typedef struct __attribute__((packed)) { + uint16_t id; /* transaction ID */ + unsigned rd : 1; /* recursion desired */ + unsigned tc : 1; /* truncation */ + unsigned aa : 1; /* authoritative answer */ + unsigned opcode : 4; /* opcode 0=std query 1=Inverse query 2=srv status + request */ + unsigned qr : 1; /* query/response */ + + unsigned rcode : 4; /* response code */ + unsigned cd : 1; /* checking disabled */ + unsigned ad : 1; /* authenticated data */ + unsigned z : 1; /* reserved set to 0 */ + unsigned ra : 1; /* recursion available */ + + uint16_t qdcount; /* # entries in question section */ + uint16_t ancount; /* # RR in answer section */ + uint16_t nscount; /* # name server RR in authority section */ + uint16_t arcount; /* # RR in additional information section */ +} dns_header; + +typedef struct __attribute__((packed)) { + uint16_t qtype; + uint16_t qclass; +} dns_question_tail; + +typedef struct __attribute__((packed)) { + uint16_t type; + uint16_t class; + uint32_t ttl; + uint16_t rdlength; + char rdata[]; +} dns_answer_tail; + +typedef enum { + DNS_QTYPE_A = 1, + DNS_QTYPE_NS = 2, + DNS_QTYPE_CNAME = 5, + DNS_QTYPE_SOA = 6, + DNS_QTYPE_PTR = 12, + DNS_QTYPE_MX = 15, + DNS_QTYPE_TXT = 16, + DNS_QTYPE_AAAA = 28, + DNS_QTYPE_RRSIG = 46, + DNS_QTYPE_ALL = 255 +} dns_qtype; + +typedef enum { + DNS_RCODE_NOERR = 0, + DNS_RCODE_FORMATERR = 1, + DNS_RCODE_SRVFAILURE = 2, + DNS_RCODE_NXDOMAIN = 3, + DNS_RCODE_QTYPENOTIMPL = 4, + DNS_RCODE_QRYREFUSED = 5 +} dns_rcode; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo.c b/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo.c new file mode 100644 index 0000000..ba50d5b --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo.c @@ -0,0 +1,334 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMP echo request (ping) scans + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <time.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "probe_modules.h" +#include "../fieldset.h" +#include "packet.h" +#include "logger.h" +#include "validate.h" + +#define ICMP_SMALLEST_SIZE 5 +#define ICMP_MAX_PAYLOAD_LEN 1458 +#define ICMP_TIMXCEED_UNREACH_HEADER_SIZE 8 + +probe_module_t module_icmp_echo; + +const char *icmp_usage_error = + "unknown ICMP probe specification (expected file:/path or text:STRING or hex:01020304)"; + +static size_t icmp_payload_len = 0; +static const size_t icmp_payload_default_len = 20; +static char *icmp_payload = NULL; + +int icmp_global_initialize(struct state_conf *conf) +{ + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp_payload = xmalloc(icmp_payload_default_len); + icmp_payload_len = icmp_payload_default_len; + return EXIT_SUCCESS; + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp_payload = strdup(c); + icmp_payload_len = strlen(icmp_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp", "could not open ICMP data file '%s'", c); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp", "unable to get size of ICMP data file '%s'", c); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP_MAX_PAYLOAD_LEN) { + log_error("icmp", "input file larger than %d bytes and will not fit on the wire (%llu bytes provided)", + ICMP_MAX_PAYLOAD_LEN, input_size); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp", "unable to read ICMP data file '%s'", c); + return EXIT_FAILURE; + } + icmp_payload = xmalloc(ICMP_MAX_PAYLOAD_LEN); + icmp_payload_len = + fread(icmp_payload, 1, ICMP_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error("icmp", "invalid hex input (length must be a multiple of 2)"); + return EXIT_FAILURE; + } + icmp_payload_len = strlen(c) / 2; + icmp_payload = xmalloc(icmp_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + free(icmp_payload); + log_error("icmp", "non-hex character: '%c'", c[i * 2]); + return EXIT_FAILURE; + } + icmp_payload[i] = (char) (n & 0xff); + } + } else { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + + if (icmp_payload_len > ICMP_MAX_PAYLOAD_LEN) { + log_error("icmp", "reducing ICMP payload must be at most %d bytes to fit on the wire (%d were provided)\n", + ICMP_MAX_PAYLOAD_LEN, icmp_payload_len); + return EXIT_FAILURE; + } + + module_icmp_echo.max_packet_length = sizeof(struct ether_header) + + sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len; + assert(module_icmp_echo.max_packet_length <= 1500); + return EXIT_SUCCESS; +} + + +int icmp_global_cleanup(__attribute__((unused)) struct state_conf *zconf, + __attribute__((unused)) struct state_send *zsend, + __attribute__((unused)) struct state_recv *zrecv) +{ + if (icmp_payload) { + free(icmp_payload); + icmp_payload = NULL; + } + + return EXIT_SUCCESS; +} + + +static int icmp_echo_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len); + make_ip_header(ip_header, IPPROTO_ICMP, len); + + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + make_icmp_header(icmp_header); + + char *payload = (char *)icmp_header + ICMP_MINLEN; + + memcpy(payload, icmp_payload, icmp_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp_echo_make_packet(void *buf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, UNUSED int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + + uint16_t icmp_idnum = validation[1] & 0xFFFF; + uint16_t icmp_seqnum = validation[2] & 0xFFFF; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + icmp_header->icmp_id = icmp_idnum; + icmp_header->icmp_seq = icmp_seqnum; + + icmp_header->icmp_cksum = 0; + icmp_header->icmp_cksum = icmp_checksum((unsigned short *)icmp_header, ICMP_MINLEN + icmp_payload_len); + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len; + ip_header->ip_len = htons(ip_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ip_len + sizeof(struct ether_header); + + return EXIT_SUCCESS; +} + +static void icmp_echo_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct icmp *icmp_header = (struct icmp *)(&iph[1]); + + fprintf(fp, + "icmp { type: %u | code: %u " + "| checksum: %#04X | id: %u | seq: %u }\n", + icmp_header->icmp_type, icmp_header->icmp_code, + ntohs(icmp_header->icmp_cksum), ntohs(icmp_header->icmp_id), + ntohs(icmp_header->icmp_seq)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static int imcp_validate_id_seq(struct icmp *icmp_h, uint32_t *validation) +{ + if (icmp_h->icmp_id != (validation[1] & 0xFFFF)) { + return PACKET_INVALID; + } + if (icmp_h->icmp_seq != (validation[2] & 0xFFFF)) { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static int icmp_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, uint32_t *validation) +{ + if (ip_hdr->ip_p != IPPROTO_ICMP) { + return PACKET_INVALID; + } + struct icmp *icmp_h = get_icmp_header(ip_hdr, len); + if (!icmp_h) { + return PACKET_INVALID; + } + if (icmp_h->icmp_type == ICMP_ECHOREPLY) { + return imcp_validate_id_seq(icmp_h, validation); + } else { + // handle unresearch/quench/redirect/timeout + struct ip *ip_inner; + size_t ip_inner_len; + int icmp_inner_valid = icmp_helper_validate( + ip_hdr, len, sizeof(struct icmp), &ip_inner, &ip_inner_len); + if (icmp_inner_valid == PACKET_INVALID) { + return PACKET_INVALID; + } + struct icmp *icmp_inner = + get_icmp_header(ip_inner, ip_inner_len); + if (!icmp_inner) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + (uint8_t *)validation); + // validate icmp id and seqnum + return imcp_validate_id_seq(icmp_inner, validation); + } +} + +static void icmp_echo_process_packet(const u_char *packet, + uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct icmp *icmp_hdr = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + fs_add_uint64(fs, "type", icmp_hdr->icmp_type); + fs_add_uint64(fs, "code", icmp_hdr->icmp_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp_hdr->icmp_id)); + fs_add_uint64(fs, "seq", ntohs(icmp_hdr->icmp_seq)); + + uint32_t hdrlen = sizeof(struct ether_header) + 4 * ip_hdr->ip_hl + 4; + + switch (icmp_hdr->icmp_type) { + case ICMP_ECHOREPLY: + fs_add_string(fs, "classification", (char *)"echoreply", 0); + fs_add_uint64(fs, "success", 1); + break; + case ICMP_UNREACH: + fs_add_string(fs, "classification", (char *)"unreach", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_SOURCEQUENCH: + fs_add_string(fs, "classification", (char *)"sourcequench", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_REDIRECT: + fs_add_string(fs, "classification", (char *)"redirect", 0); + fs_add_bool(fs, "success", 0); + break; + case ICMP_TIMXCEED: + fs_add_string(fs, "classification", (char *)"timxceed", 0); + fs_add_bool(fs, "success", 0); + break; + default: + fs_add_string(fs, "classification", (char *)"other", 0); + fs_add_bool(fs, "success", 0); + break; + } + + int datalen = len - hdrlen; + + if(datalen > 0) { + const uint8_t *data = (uint8_t *)&packet[hdrlen]; + fs_add_binary(fs, "data", (size_t)datalen, (void *)data, 0); + } else { + fs_add_null(fs, "data"); + } + +} + +static fielddef_t fields[] = { + {.name = "type", .type = "int", .desc = "icmp message type"}, + {.name = "code", .type = "int", .desc = "icmp message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "icmp id number"}, + {.name = "seq", .type = "int", .desc = "icmp sequence number"}, + {.name = "classification", + .type = "string", + .desc = "probe module classification"}, + {.name = "success", + .type = "bool", + .desc = "did probe module classify response as success"}, + {.name = "data", .type = "binary", .desc = "ICMP payload"}}; + +probe_module_t module_icmp_echo = {.name = "icmp_echoscan", + .max_packet_length = 48, + .pcap_filter = "icmp and icmp[0]!=8", + .pcap_snaplen = 96, + .port_args = 0, + .global_initialize = &icmp_global_initialize, + .close = &icmp_global_cleanup, + .thread_initialize = + &icmp_echo_init_perthread, + .make_packet = &icmp_echo_make_packet, + .print_packet = &icmp_echo_print_packet, + .process_packet = &icmp_echo_process_packet, + .validate_packet = &icmp_validate_packet, + .helptext = "Probe module that sends ICMP echo requests to hosts.\n" + "Payload of ICMP packets will consist of zeroes unless you customize it with\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 7}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo_time.c b/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo_time.c new file mode 100644 index 0000000..951b8ed --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_icmp_echo_time.c @@ -0,0 +1,248 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMP echo request (ping) scans that allows +// calculation of RTT + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <time.h> +#include <unistd.h> +#include <string.h> +#include <sys/time.h> + +#include "../../lib/includes.h" +#include "probe_modules.h" +#include "../fieldset.h" +#include "packet.h" +#include "validate.h" + +#define ICMP_SMALLEST_SIZE 5 +#define ICMP_TIMXCEED_UNREACH_HEADER_SIZE 8 + +probe_module_t module_icmp_echo_time; + +struct icmp_payload_for_rtt { + uint32_t sent_tv_sec; + uint32_t sent_tv_usec; + ipaddr_n_t dst; +}; + +static int icmp_echo_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct icmp) - 8); + make_ip_header(ip_header, IPPROTO_ICMP, len); + + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + make_icmp_header(icmp_header); + + return EXIT_SUCCESS; +} + +static int icmp_echo_make_packet(void *buf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, UNUSED int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct icmp *icmp_header = (struct icmp *)(&ip_header[1]); + struct icmp_payload_for_rtt *payload = + (struct icmp_payload_for_rtt *)(((char *)icmp_header) + 8); + + uint16_t icmp_idnum = validation[1] & 0xFFFF; + uint16_t icmp_seqnum = validation[2] & 0xFFFF; + struct timeval tv; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + icmp_header->icmp_id = icmp_idnum; + icmp_header->icmp_seq = icmp_seqnum; + + gettimeofday(&tv, NULL); + payload->sent_tv_sec = tv.tv_sec; + payload->sent_tv_usec = tv.tv_usec; + payload->dst = dst_ip; + + icmp_header->icmp_cksum = 0; + icmp_header->icmp_cksum = + icmp_checksum((unsigned short *)icmp_header, sizeof(struct icmp)); + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + ICMP_MINLEN + sizeof(struct icmp_payload_for_rtt); + ip_header->ip_len = htons(ip_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + *buf_len = ip_len + sizeof(struct ether_header); + return EXIT_SUCCESS; +} + +static void icmp_echo_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct icmp *icmp_header = (struct icmp *)(&iph[1]); + + fprintf(fp, + "icmp { type: %u | code: %u " + "| checksum: %#04X | id: %u | seq: %u }\n", + icmp_header->icmp_type, icmp_header->icmp_code, + ntohs(icmp_header->icmp_cksum), ntohs(icmp_header->icmp_id), + ntohs(icmp_header->icmp_seq)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static int icmp_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation) +{ + if (ip_hdr->ip_p != IPPROTO_ICMP) { + return 0; + } + if (((uint32_t)4 * ip_hdr->ip_hl + ICMP_SMALLEST_SIZE) > len) { + // buffer not large enough to contain expected icmp header + return 0; + } + struct icmp *icmp_h = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t icmp_idnum = icmp_h->icmp_id; + uint16_t icmp_seqnum = icmp_h->icmp_seq; + // ICMP validation is tricky: for some packet types, we must look inside + // the payload + if (icmp_h->icmp_type == ICMP_TIMXCEED || + icmp_h->icmp_type == ICMP_UNREACH) { + // Should have 16B TimeExceeded/Dest_Unreachable header + + // original IP header + 1st 8B of original ICMP frame + if ((4 * ip_hdr->ip_hl + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + + sizeof(struct ip)) > len) { + return 0; + } + struct ip *ip_inner = (struct ip *)((char *)icmp_h + 8); + if (((uint32_t)4 * ip_hdr->ip_hl + + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + 4 * ip_inner->ip_hl + + 8 /*1st 8 bytes of original*/) > len) { + return 0; + } + struct icmp *icmp_inner = + (struct icmp *)((char *)ip_inner + 4 * ip_hdr->ip_hl); + // Regenerate validation and icmp id based off inner payload + icmp_idnum = icmp_inner->icmp_id; + icmp_seqnum = icmp_inner->icmp_seq; + *src_ip = ip_inner->ip_dst.s_addr; + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + (uint8_t *)validation); + } + // validate icmp id and seqnum + if (icmp_idnum != (validation[1] & 0xFFFF)) { + return 0; + } + if (icmp_seqnum != (validation[2] & 0xFFFF)) { + return 0; + } + + return 1; +} + +static void icmp_echo_process_packet(const u_char *packet, + UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + struct icmp *icmp_hdr = + (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + fs_add_uint64(fs, "type", icmp_hdr->icmp_type); + fs_add_uint64(fs, "code", icmp_hdr->icmp_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp_hdr->icmp_id)); + fs_add_uint64(fs, "seq", ntohs(icmp_hdr->icmp_seq)); + + struct icmp_payload_for_rtt *payload = + (struct icmp_payload_for_rtt *)(((char *)icmp_hdr) + 8); + fs_add_uint64(fs, "sent_timestamp_ts", (uint64_t)payload->sent_tv_sec); + fs_add_uint64(fs, "sent_timestamp_us", (uint64_t)payload->sent_tv_usec); + fs_add_uint64(fs, "dst_raw", (uint64_t)payload->dst); + + switch (icmp_hdr->icmp_type) { + case ICMP_ECHOREPLY: + fs_add_string(fs, "classification", (char *)"echoreply", 0); + fs_add_uint64(fs, "success", 1); + break; + case ICMP_UNREACH: + fs_add_string(fs, "classification", (char *)"unreach", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_SOURCEQUENCH: + fs_add_string(fs, "classification", (char *)"sourcequench", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_REDIRECT: + fs_add_string(fs, "classification", (char *)"redirect", 0); + fs_add_uint64(fs, "success", 0); + break; + case ICMP_TIMXCEED: + fs_add_string(fs, "classification", (char *)"timxceed", 0); + fs_add_uint64(fs, "success", 0); + break; + default: + fs_add_string(fs, "classification", (char *)"other", 0); + fs_add_uint64(fs, "success", 0); + break; + } +} + +static fielddef_t fields[] = { + {.name = "type", .type = "int", .desc = "icmp message type"}, + {.name = "code", .type = "int", .desc = "icmp message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "icmp id number"}, + {.name = "seq", .type = "int", .desc = "icmp sequence number"}, + {.name = "sent_timestamp_ts", + .type = "int", + .desc = "timestamp of sent probe in seconds since Epoch"}, + {.name = "sent_timestamp_us", + .type = "int", + .desc = "microsecond part of sent timestamp"}, + {.name = "dst-raw", + .type = "int", + .desc = "raw destination IP address of sent probe"}, + {.name = "classification", + .type = "string", + .desc = "probe module classification"}, + {.name = "success", + .type = "int", + .desc = "did probe module classify response as success"}}; + +probe_module_t module_icmp_echo_time = { + .name = "icmp_echo_time", + .max_packet_length = 62, + .pcap_filter = "icmp and icmp[0]!=8", + .pcap_snaplen = 96, + .port_args = 0, + .thread_initialize = &icmp_echo_init_perthread, + .make_packet = &icmp_echo_make_packet, + .print_packet = &icmp_echo_print_packet, + .process_packet = &icmp_echo_process_packet, + .validate_packet = &icmp_validate_packet, + .close = NULL, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = 9}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.c b/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.c new file mode 100644 index 0000000..20367af --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.c @@ -0,0 +1,256 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "probe_modules.h" +#include "module_udp.h" +#include "module_ntp.h" +#include "packet.h" +#include "logger.h" + +#define MAX_NTP_PAYLOAD_LEN 1472 +#define ICMP_UNREACH_HEADER_SIZE 8 + +probe_module_t module_ntp; + +static int num_ports; + + +int ntp_global_initialize(struct state_conf *conf) { + num_ports = conf->source_port_last - conf->source_port_first + 1; + return udp_global_initialize(conf); +} + +int ntp_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, zconf.target_port); +} + +void ntp_process_packet(const u_char *packet, + UNUSED uint32_t len, fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + uint64_t temp64; + uint8_t temp8; + uint32_t temp32; + + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = + (struct udphdr *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + uint8_t *ptr = (uint8_t *)&udp[1]; + + fs_add_string(fs, "classification", (char *)"ntp", 0); + fs_add_bool(fs, "success", 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + + if (len > 90) { + temp8 = *((uint8_t *)ptr); + fs_add_uint64(fs, "LI_VN_MODE", temp8); + temp8 = *((uint8_t *)ptr + 1); + fs_add_uint64(fs, "stratum", temp8); + temp8 = *((uint8_t *)ptr + 2); + fs_add_uint64(fs, "poll", temp8); + temp8 = *((uint8_t *)ptr + 3); + fs_add_uint64(fs, "precision", temp8); + + temp32 = *((uint32_t *)ptr + 4); + fs_add_uint64(fs, "root_delay", temp32); + temp32 = *((uint32_t *)ptr + 8); + fs_add_uint64(fs, "root_dispersion", temp32); + temp32 = *((uint32_t *)ptr + 12); + fs_add_uint64(fs, "reference_clock_identifier", temp32); + + temp64 = *((uint64_t *)ptr + 16); + fs_add_uint64(fs, "reference_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 24); + fs_add_uint64(fs, "originate_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 32); + fs_add_uint64(fs, "receive_timestamp", temp64); + temp64 = *((uint64_t *)ptr + 39); + fs_add_uint64(fs, "transmit_timestamp", temp64); + } else { + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct icmp *icmp = + (struct icmp *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + struct ip *ip_inner = + (struct ip *)((char *)icmp + ICMP_UNREACH_HEADER_SIZE); + + fs_modify_string(fs, "saddr", + make_ip_str(ip_inner->ip_dst.s_addr), 1); + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_string(fs, "icmp_responder", + make_ip_str(ip_hdr->ip_src.s_addr), 1); + fs_add_uint64(fs, "icmp_type", icmp->icmp_type); + fs_add_uint64(fs, "icmp_code", icmp->icmp_code); + fs_add_null(fs, "icmp_unreach_str"); + + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + fs_add_null(fs, "LI_VN_MODE"); + fs_add_null(fs, "stratum"); + fs_add_null(fs, "poll"); + fs_add_null(fs, "precision"); + fs_add_null(fs, "root_delay"); + fs_add_null(fs, "root_dispersion"); + fs_add_null(fs, "reference_clock_identifier"); + fs_add_null(fs, "reference_timestamp"); + fs_add_null(fs, "originate_timestamp"); + fs_add_null(fs, "receive_timestamp"); + fs_add_null(fs, "transmit_timestamp"); + } +} + +int ntp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, void **arg) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + sizeof(struct ntphdr)); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + struct ntphdr *ntp_header = (struct ntphdr *)(&udp_header[1]); + ntp_header->LI_VN_MODE = 227; + len = sizeof(struct udphdr) + sizeof(struct ntphdr); + + make_udp_header(udp_header, zconf.target_port, len); + + // TODO(dadrian): Should this have a payload? It was being set incorrectly. + size_t header_len = sizeof(struct ether_header) + + sizeof(struct ip) + + sizeof(struct udphdr) + + sizeof(struct ntphdr); + module_ntp.max_packet_length = header_len; + + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg = aes; + + return EXIT_SUCCESS; +} + +void ntp_print_packet(FILE *fp, void *packet) +{ + + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(iph + 4 * iph->ip_hl); + struct ntphdr *ntph = (struct ntphdr *)&udph[1]; + fprintf(fp, "ntp { LI_VN_MODE: %u | stratum: %u | poll: %u }\n", + ntph->LI_VN_MODE, ntph->stratum, ntph->poll); + fprintf(fp, "udp { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +static fielddef_t fields[] = { + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + ICMP_FIELDSET_FIELDS, + {.name = "LI_VN_MODE", + .type = "int", + .desc = "leap indication, version number, mode"}, + {.name = "stratum", .type = "int", .desc = "stratum"}, + {.name = "poll", .type = "int", .desc = "poll"}, + {.name = "precision", .type = "int", .desc = "precision"}, + {.name = "root_delay", .type = "int", .desc = "root delay"}, + {.name = "root_dispersion", .type = "int", .desc = "root dispersion"}, + {.name = "reference_clock_identifier", + .type = "int", + .desc = "code identifying clock reference"}, + {.name = "reference_timestamp", + .type = "int", + .desc = "local time at which local clock was last set or corrected"}, + {.name = "originate_timestamp", + .type = "int", + .desc = "local time at which request deparated client for service"}, + {.name = "receive_timestamp", + .type = "int", + .desc = "local time at which request arrvied at service host"}, + {.name = "transmit_timestamp", + .type = "int", + .desc = "local time which reply departed service host for client"}, +}; + +probe_module_t module_ntp = {.name = "ntp", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .thread_initialize = &ntp_init_perthread, + .global_initialize = &ntp_global_initialize, + .make_packet = &udp_make_packet, + .print_packet = &ntp_print_packet, + .validate_packet = &ntp_validate_packet, + .process_packet = &ntp_process_packet, + .close = &udp_global_cleanup, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.h b/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.h new file mode 100644 index 0000000..903a0f5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_ntp.h @@ -0,0 +1,38 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "../state.h" +#include "../fieldset.h" + +#ifndef MODULE_NTP_H +#define MODULE_NTP_H + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> + +struct __attribute__((__packed__)) ntphdr { // typedef + uint8_t LI_VN_MODE; + uint8_t stratum; + uint8_t poll; + uint8_t precision; + uint32_t root_delay; + uint32_t root_dispersion; + uint32_t ref_ID; + uint64_t reference_timestamp; + uint64_t origin_timestamp; + uint64_t receive_timestamp; + uint64_t transmit_timestamp; + // uint32_t key_ID; + // uint64_t dgst_1; + // uint64_t dgst_2; +}; + +#endif diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synackscan.c b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synackscan.c new file mode 100644 index 0000000..f52220c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synackscan.c @@ -0,0 +1,224 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "validate.h" +#include "module_tcp_synscan.h" + +#define ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN 24 +#define ZMAP_TCP_SYNACKSCAN_PACKET_LEN 58 + +probe_module_t module_tcp_synackscan; +static uint32_t num_ports; + +static int synackscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int synackscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, dst_port, TH_SYN | TH_ACK); + set_mss_option(tcp_header); + return EXIT_SUCCESS; +} + +static int synackscan_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + uint32_t tcp_seq = validation[0]; + uint32_t tcp_ack = + validation[2]; // get_src_port() below uses validation 1 internally. + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + tcp_header->th_sport = + htons(get_src_port(num_ports, probe_num, validation)); + tcp_header->th_seq = tcp_seq; + tcp_header->th_ack = tcp_ack; + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp_checksum(ZMAP_TCP_SYNACKSCAN_TCP_HEADER_LEN, ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + *buf_len = ZMAP_TCP_SYNACKSCAN_PACKET_LEN; + + return EXIT_SUCCESS; +} + +static int synackscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, + uint32_t *validation) +{ + + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + if (!tcp) { + return PACKET_INVALID; + } + uint16_t sport = ntohs(tcp->th_sport); + uint16_t dport = ntohs(tcp->th_dport); + // validate source port + if (sport != zconf.target_port) { + return PACKET_INVALID; + } + // validate destination port + if (!check_dst_port(dport, num_ports, validation)) { + return PACKET_INVALID; + } + // check whether we'll ever send to this IP during the scan + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + // We handle RST packets different than all other packets + if (tcp->th_flags & TH_RST) { + // A RST packet must have either: + // 1) resp(ack) == sent(seq) + 1, or + // 2) resp(seq) == sent(ack), or + // 3) resp(seq) == sent(ack) + 1 + // All other cases are a failure. + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1 && + htonl(tcp->th_seq) != htonl(validation[2]) && + htonl(tcp->th_seq) != (htonl(validation[2]) + 1)) { + return PACKET_INVALID; + } + } else { + // For non RST packets, we must have resp(ack) == sent(seq) + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct udphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + return PACKET_INVALID; + } + struct tcphdr *tcp = get_tcp_header(ip_inner, ip_inner_len); + if (!tcp) { + return PACKET_INVALID; + } + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + uint16_t sport = ntohs(tcp->th_sport); + uint16_t dport = ntohs(tcp->th_dport); + if (dport != zconf.target_port) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + (uint8_t *)validation); + if (!check_dst_port(sport, num_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static void synackscan_process_packet(const u_char *packet, + UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_constchar(fs, "classification", "rst"); + } else { // SYNACK packet + fs_add_constchar(fs, "classification", "synack"); + } + fs_add_bool(fs, "success", 1); + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + // tcp + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "seqnum"); + fs_add_null(fs, "acknum"); + fs_add_null(fs, "window"); + // global + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + // icmp + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_tcp_synackscan = { + .name = "tcp_synackscan", + .max_packet_length = ZMAP_TCP_SYNACKSCAN_PACKET_LEN, + .pcap_filter = "(tcp && tcp[13] & 4 != 0 || tcp[13] == 18) || icmp", + .pcap_snaplen = 96, + .port_args = 1, + .global_initialize = &synackscan_global_initialize, + .thread_initialize = &synackscan_init_perthread, + .make_packet = &synackscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synackscan_process_packet, + .validate_packet = &synackscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYNACK packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a failure and a reset packet " + "is considered a success.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]) +}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.c b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.c new file mode 100644 index 0000000..ae7f71a --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.c @@ -0,0 +1,237 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "validate.h" + +#define ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN 24 +#define ZMAP_TCP_SYNSCAN_PACKET_LEN 58 + +probe_module_t module_tcp_synscan; + +static uint16_t num_ports; +static port_h_t target_port; + +static int synscan_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + target_port = state->target_port; + return EXIT_SUCCESS; +} + +static int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + UNUSED void **arg_ptr) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t len = htons(sizeof(struct ip) + ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN); + make_ip_header(ip_header, IPPROTO_TCP, len); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + make_tcp_header(tcp_header, dst_port, TH_SYN); + set_mss_option(tcp_header); + return EXIT_SUCCESS; +} + +static int synscan_make_packet(void *buf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct tcphdr *tcp_header = (struct tcphdr *)(&ip_header[1]); + uint32_t tcp_seq = validation[0]; + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + + port_h_t sport = get_src_port(num_ports, probe_num, validation); + tcp_header->th_sport = htons(sport); + tcp_header->th_seq = tcp_seq; + // checksum value must be zero when calculating packet's checksum + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp_checksum(ZMAP_TCP_SYNSCAN_TCP_HEADER_LEN, ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + // checksum value must be zero when calculating packet's checksum + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + *buf_len = ZMAP_TCP_SYNSCAN_PACKET_LEN; + return EXIT_SUCCESS; +} + +// not static because used by synack scan +void synscan_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct tcphdr *tcph = (struct tcphdr *)&iph[1]; + fprintf(fp, + "tcp { source: %u | dest: %u | seq: %u | checksum: %#04X }\n", + ntohs(tcph->th_sport), ntohs(tcph->th_dport), + ntohl(tcph->th_seq), ntohs(tcph->th_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + + +static int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation) +{ + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + if (!tcp) { + return PACKET_INVALID; + } + port_h_t sport = ntohs(tcp->th_sport); + port_h_t dport = ntohs(tcp->th_dport); + // validate source port + if (sport != target_port) { + return PACKET_INVALID; + } + // validate destination port + if (!check_dst_port(dport, num_ports, validation)) { + return PACKET_INVALID; + } + // check whether we'll ever send to this IP during the scan + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + // We treat RST packets different from non RST packets + if (tcp->th_flags & TH_RST) { + // For RST packets, recv(ack) == sent(seq) + 0 or + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) && + htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } else { + // For non RST packets, recv(ack) == sent(seq) + 1 + if (htonl(tcp->th_ack) != htonl(validation[0]) + 1) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct tcphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + return PACKET_INVALID; + } + struct tcphdr *tcp = get_tcp_header(ip_inner, ip_inner_len); + if (!tcp) { + return PACKET_INVALID; + } + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port. Note this is *different* + // than the logic above because we're validating the probe packet + // rather than the response packet + port_h_t sport = ntohs(tcp->th_sport); + port_h_t dport = ntohs(tcp->th_dport); + if (dport != target_port) { + return PACKET_INVALID; + } + validate_gen(ip_hdr->ip_dst.s_addr, ip_inner->ip_dst.s_addr, + (uint8_t *)validation); + if (!check_dst_port(sport, num_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +static void synscan_process_packet(const u_char *packet, + UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = get_ip_header(packet, len); + assert(ip_hdr); + if (ip_hdr->ip_p == IPPROTO_TCP) { + struct tcphdr *tcp = get_tcp_header(ip_hdr, len); + assert(tcp); + fs_add_uint64(fs, "sport", (uint64_t)ntohs(tcp->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t)ntohs(tcp->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t)ntohl(tcp->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t)ntohl(tcp->th_ack)); + fs_add_uint64(fs, "window", (uint64_t)ntohs(tcp->th_win)); + if (tcp->th_flags & TH_RST) { // RST packet + fs_add_constchar(fs, "classification", "rst"); + fs_add_bool(fs, "success", 0); + } else { // SYNACK packet + fs_add_constchar(fs, "classification", "synack"); + fs_add_bool(fs, "success", 1); + } + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + // tcp + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "seqnum"); + fs_add_null(fs, "acknum"); + fs_add_null(fs, "window"); + // global + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + // icmp + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window"}, + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_tcp_synscan = { + .name = "tcp_synscan", + .max_packet_length = ZMAP_TCP_SYNSCAN_PACKET_LEN, + .pcap_filter = "(tcp && tcp[13] & 4 != 0 || tcp[13] == 18) || icmp", + .pcap_snaplen = 96, + .port_args = 1, + .global_initialize = &synscan_global_initialize, + .thread_initialize = &synscan_init_perthread, + .make_packet = &synscan_make_packet, + .print_packet = &synscan_print_packet, + .process_packet = &synscan_process_packet, + .validate_packet = &synscan_validate_packet, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]) +}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.h b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.h new file mode 100644 index 0000000..9b707c9 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_tcp_synscan.h @@ -0,0 +1,42 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +int synscan_global_initialize(struct state_conf *state); + +int synscan_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + UNUSED void **arg_ptr); + +int synscan_make_packet(void *buf, ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + UNUSED void *arg); + +void synscan_print_packet(FILE *fp, void *packet); + +int synscan_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, + uint32_t *validation); + +void synscan_process_packet(const u_char *packet, + UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation); diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_udp.c b/rdns_scan/zmap4rdns/src/probe_modules/module_udp.c new file mode 100644 index 0000000..484798b --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_udp.c @@ -0,0 +1,852 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* send module for performing arbitrary UDP scans */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include <errno.h> + +#include "../../lib/blocklist.h" +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "../../lib/lockfd.h" +#include "logger.h" +#include "probe_modules.h" +#include "packet.h" +#include "aesrand.h" +#include "state.h" +#include "module_udp.h" + +#define MAX_UDP_PAYLOAD_LEN 1472 +#define ICMP_HEADER_SIZE 8 + +static uint8_t *udp_fixed_payload = NULL; +static size_t udp_fixed_payload_len = 0; + +static udp_payload_template_t *udp_template = NULL; + +const char *udp_usage_error = + "unknown UDP probe specification (expected file:/path or text:STRING or hex:01020304 or template:/path or template-fields)"; + +const unsigned char *charset_alphanum = + (unsigned char + *)"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +const unsigned char *charset_alpha = + (unsigned char *)"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const unsigned char *charset_digit = (unsigned char *)"0123456789"; +const unsigned char charset_all[257] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00}; + +static int num_ports; + +probe_module_t module_udp; + +// Field definitions for template parsing and displaying usage +static uint32_t udp_num_template_field_types = 12; +static udp_payload_field_type_def_t udp_payload_template_fields[] = { + {.name = "SADDR_N", + .ftype = UDP_SADDR_N, + .max_length = 4, + .desc = "Source IP address in network byte order"}, + {.name = "SADDR", + .ftype = UDP_SADDR_A, + .max_length = 15, + .desc = "Source IP address in dotted-quad format"}, + {.name = "DADDR_N", + .ftype = UDP_DADDR_N, + .max_length = 4, + .desc = "Destination IP address in network byte order"}, + {.name = "DADDR", + .ftype = UDP_DADDR_A, + .max_length = 15, + .desc = "Destination IP address in dotted-quad format"}, + {.name = "SPORT_N", + .ftype = UDP_SPORT_N, + .max_length = 2, + .desc = "UDP source port in network byte order"}, + {.name = "SPORT", + .ftype = UDP_SPORT_A, + .max_length = 5, + .desc = "UDP source port in ascii format"}, + {.name = "DPORT_N", + .ftype = UDP_DPORT_N, + .max_length = 2, + .desc = "UDP destination port in network byte order"}, + {.name = "DPORT", + .ftype = UDP_DPORT_A, + .max_length = 5, + .desc = "UDP destination port in ascii format"}, + {.name = "RAND_BYTE", + .ftype = UDP_RAND_BYTE, + .max_length = 0, + .desc = "Random bytes from 0-255"}, + {.name = "RAND_DIGIT", + .ftype = UDP_RAND_DIGIT, + .max_length = 0, + .desc = "Random digits from 0-9"}, + {.name = "RAND_ALPHA", + .ftype = UDP_RAND_ALPHA, + .max_length = 0, + .desc = "Random mixed-case letters (a-z)"}, + {.name = "RAND_ALPHANUM", + .ftype = UDP_RAND_ALPHANUM, + .max_length = 0, + .desc = "Random mixed-case letters (a-z) and numbers"}}; + + +void udp_set_num_ports(int x) { num_ports = x; } + +int udp_global_initialize(struct state_conf *conf) +{ + uint32_t udp_template_max_len = 0; + num_ports = conf->source_port_last - conf->source_port_first + 1; + + if (!conf->probe_args) { + log_error( + "udp", "%s", + "--probe-args are required, run --probe-module=udp --help for a longer description of the arguments"); + return EXIT_FAILURE; + } + + const char *args = conf->probe_args; + if (strcmp(args, "template-fields") == 0) { + lock_file(stderr); + fprintf( + stderr, "%s", + "List of allowed UDP template fields (name: description)\n\n"); + for (uint32_t i = 0; i < udp_num_template_field_types; ++i) { + fprintf(stderr, "%s: %s\n", + udp_payload_template_fields[i].name, + udp_payload_template_fields[i].desc); + } + fprintf(stderr, "%s\n", ""); + fflush(stderr); + unlock_file(stderr); + exit(0); + } + + const char *c = strchr(args, ':'); + if (!c) { + log_fatal("udp", udp_usage_error); + } + size_t arg_name_len = c - args; + c++; + if (strncmp(args, "text", arg_name_len) == 0) { + udp_fixed_payload = (uint8_t*) strdup(c); + udp_fixed_payload_len = strlen(c); + } else if (strncmp(args, "file", arg_name_len) == 0) { + udp_fixed_payload = xmalloc(MAX_UDP_PAYLOAD_LEN); + FILE *f = fopen(c, "rb"); + if (!f) { + log_fatal("udp", "could not open UDP data file '%s'\n", + c); + } + udp_fixed_payload_len = + fread(udp_fixed_payload, 1, MAX_UDP_PAYLOAD_LEN, f); + fclose(f); + } else if (strncmp(args, "template", arg_name_len) == 0) { + uint8_t in[MAX_UDP_PAYLOAD_LEN]; + FILE *f = fopen(c, "rb"); + if (!f) { + log_fatal("udp", "could not open UDP data file '%s'\n", + c); + } + size_t in_len = fread(in, 1, MAX_UDP_PAYLOAD_LEN, f); + fclose(f); + + udp_template = udp_template_load(in, in_len, &udp_template_max_len); + module_udp.make_packet = udp_make_templated_packet; + } else if (strncmp(args, "hex", arg_name_len) == 0) { + udp_fixed_payload_len = strlen(c) / 2; + udp_fixed_payload = xmalloc(udp_fixed_payload_len); + + unsigned int n; + for (size_t i = 0; i < udp_fixed_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_fatal("udp", "non-hex character: '%c'", + c[i * 2]); + } + udp_fixed_payload[i] = (n & 0xff); + } + } else { + log_fatal("udp", udp_usage_error); + } + + if (udp_fixed_payload_len > MAX_UDP_PAYLOAD_LEN) { + log_warn("udp", + "warning: reducing fixed UDP payload to %d " + "bytes (from %d) to fit on the wire\n", + MAX_UDP_PAYLOAD_LEN, udp_fixed_payload_len); + udp_fixed_payload_len = MAX_UDP_PAYLOAD_LEN; + } + + size_t header_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + if (udp_fixed_payload_len > 0) { + module_udp.max_packet_length = + header_len + udp_fixed_payload_len; + } else if (udp_template_max_len > 0) { + module_udp.max_packet_length = + header_len + udp_template_max_len; + } + assert(module_udp.max_packet_length); + assert(module_udp.max_packet_length <= MAX_PACKET_SIZE); + return EXIT_SUCCESS; +} + +int udp_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv) +{ + if (udp_fixed_payload) { + free(udp_fixed_payload); + udp_fixed_payload = NULL; + } + if (udp_template) { + udp_template_free(udp_template); + udp_template = NULL; + } + return EXIT_SUCCESS; +} + +int udp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED port_h_t dst_port, + void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + uint16_t ip_len = + htons(sizeof(struct ip) + sizeof(struct udphdr) + udp_fixed_payload_len); + make_ip_header(ip_header, IPPROTO_UDP, ip_len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + uint16_t udp_len = sizeof(struct udphdr) + udp_fixed_payload_len; + make_udp_header(udp_header, zconf.target_port, udp_len); + + if (udp_fixed_payload) { + void *payload = &udp_header[1]; + memcpy(payload, udp_fixed_payload, udp_fixed_payload_len); + } + + // Seed our random number generator with the global generator + uint32_t seed = aesrand_getword(zconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg_ptr = aes; + + return EXIT_SUCCESS; +} + +int udp_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, uint8_t ttl, uint32_t *validation, + int probe_num, UNUSED void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + size_t headers_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + // Output the total length of the packet + *buf_len = headers_len + udp_fixed_payload_len; + return EXIT_SUCCESS; +} + +int udp_make_templated_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, void *arg) +{ + struct ether_header *eth_header = (struct ether_header *)buf; + struct ip *ip_header = (struct ip *)(ð_header[1]); + struct udphdr *udp_header = (struct udphdr *)&ip_header[1]; + size_t headers_len = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr); + + ip_header->ip_src.s_addr = src_ip; + ip_header->ip_dst.s_addr = dst_ip; + ip_header->ip_ttl = ttl; + udp_header->uh_sport = + htons(get_src_port(num_ports, probe_num, validation)); + + char *payload = (char *)&udp_header[1]; + memset(payload, 0, MAX_UDP_PAYLOAD_LEN); + + // Grab our random number generator + aesrand_t *aes = (aesrand_t *)arg; + + // The buf is a stack var of our caller of size MAX_PACKET_SIZE + // Recalculate the payload using the loaded template + int payload_len = + udp_template_build(udp_template, payload, MAX_UDP_PAYLOAD_LEN, + ip_header, udp_header, aes); + + // If success is zero, the template output was truncated + if (payload_len <= 0) { + log_fatal("udp", + "UDP payload template generated an empty payload"); + } + + // Update the IP and UDP headers to match the new payload length + ip_header->ip_len = + htons(sizeof(struct ip) + sizeof(struct udphdr) + payload_len); + udp_header->uh_ulen = ntohs(sizeof(struct udphdr) + payload_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = zmap_ip_checksum((unsigned short *)ip_header); + + // Recalculate the total length of the packet + *buf_len = headers_len + payload_len; + return EXIT_SUCCESS; +} + +void udp_print_packet(FILE *fp, void *packet) +{ + struct ether_header *ethh = (struct ether_header *)packet; + struct ip *iph = (struct ip *)ðh[1]; + struct udphdr *udph = (struct udphdr *)(&iph[1]); + fprintf(fp, "udp { source: %u | dest: %u | checksum: %#04X }\n", + ntohs(udph->uh_sport), ntohs(udph->uh_dport), + ntohs(udph->uh_sum)); + fprintf_ip_header(fp, iph); + fprintf_eth_header(fp, ethh); + fprintf(fp, PRINT_PACKET_SEP); +} + +void udp_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + fs_add_constchar(fs, "classification", "udp"); + fs_add_bool(fs, "success", 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_uint64(fs, "udp_pkt_size", ntohs(udp->uh_ulen)); + // Verify that the UDP length is big enough for the header and + // at least one byte + uint16_t data_len = ntohs(udp->uh_ulen); + if (data_len > sizeof(struct udphdr)) { + uint32_t overhead = + (sizeof(struct udphdr) + (ip_hdr->ip_hl * 4)); + uint32_t max_rlen = len - overhead; + uint32_t max_ilen = ntohs(ip_hdr->ip_len) - overhead; + + // Verify that the UDP length is inside of our received + // buffer + if (data_len > max_rlen) { + data_len = max_rlen; + } + // Verify that the UDP length is inside of our IP packet + if (data_len > max_ilen) { + data_len = max_ilen; + } + fs_add_binary(fs, "data", data_len, (void *)&udp[1], 0); + // Some devices reply with a zero UDP length but still + // return data, ignore the data + } else { + fs_add_null(fs, "data"); + } + fs_add_null_icmp(fs); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_constchar(fs, "classification", "icmp"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + fs_add_null_icmp(fs); + } +} + +int udp_validate_packet(const struct ip *ip_hdr, uint32_t len, uint32_t *src_ip, + uint32_t *validation) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, NO_SRC_PORT_VALIDATION); +} + +// Do very basic validation that this is an ICMP response to a packet we sent +// Find the application layer packet that was originally sent and give it back +// to the caller to do additional validation (e.g., correct TCP destination +// port) + +int udp_do_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation, + int num_ports, int expected_port) +{ + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = get_udp_header(ip_hdr, len); + if (!udp) { + return PACKET_INVALID; + } + uint16_t dport = ntohs(udp->uh_dport); + if (!check_dst_port(dport, num_ports, validation)) { + return PACKET_INVALID; + } + if (!blocklist_is_allowed(*src_ip)) { + return PACKET_INVALID; + } + if (expected_port != NO_SRC_PORT_VALIDATION) { + uint16_t ep = (uint16_t)expected_port; + uint16_t sport = ntohs(udp->uh_sport); + if (sport != ep) { + return PACKET_INVALID; + } + } + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + struct ip *ip_inner; + size_t ip_inner_len; + if (icmp_helper_validate(ip_hdr, len, sizeof(struct udphdr), + &ip_inner, + &ip_inner_len) == PACKET_INVALID) { + } + struct udphdr *udp = get_udp_header(ip_inner, ip_inner_len); + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + uint16_t dport = ntohs(udp->uh_dport); + uint16_t sport = ntohs(udp->uh_sport); + if (dport != zconf.target_port) { + return PACKET_INVALID; + } + if (!check_dst_port(sport, num_ports, validation)) { + return PACKET_INVALID; + } + } else { + return PACKET_INVALID; + } + return PACKET_VALID; +} + +// Add a new field to the template +void udp_template_add_field(udp_payload_template_t *t, + udp_payload_field_type_t ftype, unsigned int length, + char *data) +{ + udp_payload_field_t *c; + + t->fcount++; + t->fields = + xrealloc(t->fields, sizeof(udp_payload_field_t) * t->fcount); + t->fields[t->fcount - 1] = xmalloc(sizeof(udp_payload_field_t)); + c = t->fields[t->fcount - 1]; + assert(c); + c->ftype = ftype; + c->length = length; + c->data = data; +} + +// Free all buffers held by the payload template, including its own +void udp_template_free(udp_payload_template_t *t) +{ + for (unsigned int x = 0; x < t->fcount; x++) { + if (t->fields[x]->data) { + free(t->fields[x]->data); + t->fields[x]->data = NULL; + } + free(t->fields[x]); + t->fields[x] = NULL; + } + free(t->fields); + t->fields = NULL; + t->fcount = 0; + free(t); +} + +int udp_random_bytes(char *dst, int len, const unsigned char *charset, + int charset_len, aesrand_t *aes) +{ + int i; + for (i = 0; i < len; i++) { + *dst++ = + charset[(aesrand_getword(aes) & 0xFFFFFFFF) % charset_len]; + } + return i; +} + +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes) +{ + udp_payload_field_t *c; + char *p; + char *max; + char tmp[256]; + int full = 0; + unsigned int x, y; + uint32_t *u32; + uint16_t *u16; + + max = out + len; + p = out; + + for (x = 0; x < t->fcount; x++) { + c = t->fields[x]; + + // Exit the processing loop if our packet buffer would overflow + if (p + c->length >= max) { + full = 1; + return 0; + } + + switch (c->ftype) { + // These fields have a specified output length value + + case UDP_DATA: + if (!(c->data && c->length)) + break; + memcpy(p, c->data, c->length); + p += c->length; + break; + + case UDP_RAND_DIGIT: + p += udp_random_bytes(p, c->length, charset_digit, 10, + aes); + break; + + case UDP_RAND_ALPHA: + p += udp_random_bytes(p, c->length, charset_alpha, 52, + aes); + break; + + case UDP_RAND_ALPHANUM: + p += udp_random_bytes(p, c->length, charset_alphanum, + 62, aes); + break; + + case UDP_RAND_BYTE: + p += udp_random_bytes(p, c->length, charset_all, 256, + aes); + break; + + // These fields need to calculate size on their own + + // TODO: Condense these case statements to remove redundant code + case UDP_SADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *)&ip_hdr->ip_src, tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_DADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *)&ip_hdr->ip_dst, tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_SADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + + u32 = (uint32_t *)p; + *u32 = ip_hdr->ip_src.s_addr; + p += 4; + break; + + case UDP_DADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + u32 = (uint32_t *)p; + *u32 = ip_hdr->ip_dst.s_addr; + p += 4; + break; + + case UDP_SPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *)p; + *u16 = udp_hdr->uh_sport; + p += 2; + break; + + case UDP_DPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *)p; + *u16 = udp_hdr->uh_sport; + p += 2; + break; + + case UDP_SPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + + case UDP_DPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + } + + // Bail out if our packet buffer would overflow + if (full == 1) { + return 0; + } + } + + return p - out; +} + +// Convert a string field name to a field type, parsing any specified length +// value +int udp_template_field_lookup(const char *vname, udp_payload_field_t *c) +{ + static const size_t fcount = sizeof(udp_payload_template_fields) / + sizeof(udp_payload_template_fields[0]); + size_t vname_len = strlen(vname); + size_t type_name_len = vname_len; + const char *param = strstr(vname, "="); + if (param) { + type_name_len = param - vname; + param++; + } + + // Most field types treat their parameter as a generator output length + // unless it is ignored (ADDR, PORT, etc). + long olen = 0; + if (param && !*param) { + log_fatal("udp", "invalid template: field spec %s is invalid (missing length)", vname); + } + if (param) { + char *end = NULL; + errno = 0; + olen = strtol(param, &end, 10); + if (errno) { + log_fatal("udp", "invalid template: unable to read length from %s: %s", vname, strerror(errno)); + } + if (!end || end != vname + vname_len) { + log_fatal("udp", "invalid template: unable to read length from %s", vname); + } + if (olen < 0 || olen > MAX_UDP_PAYLOAD_LEN) { + log_fatal("udp", "invalid template: field size %d is larger than the max (%d)", olen, MAX_UDP_PAYLOAD_LEN); + } + } + + // Find a field that matches the + for (unsigned int f = 0; f < fcount; f++) { + const udp_payload_field_type_def_t* ftype = &udp_payload_template_fields[f]; + if (strncmp(vname, ftype->name, type_name_len) == 0 && strlen(ftype->name) == type_name_len) { + c->ftype = ftype->ftype; + c->length = ftype->max_length ? ftype->max_length : (size_t) olen; + c->data = NULL; + return 1; + } + } + + // No match, skip and treat it as a data field + return 0; +} + +// Allocate a payload template and populate it by parsing a template file as a +// binary buffer +udp_payload_template_t *udp_template_load(uint8_t *buf, uint32_t buf_len, + uint32_t *max_pkt_len) +{ + udp_payload_template_t *t = xmalloc(sizeof(udp_payload_template_t)); + uint32_t _max_pkt_len = 0; + + // The last $ we encountered outside of a field specifier + uint8_t *dollar = NULL; + + // The last { we encountered outside of a field specifier + uint8_t *lbrack = NULL; + + // Track the start pointer of a data field (static) + uint8_t *s = buf; + + // Track the index into the template + uint8_t *p = buf; + + char *tmp; + unsigned int tlen; + + udp_payload_field_t c; + + t->fcount = 0; + t->fields = NULL; + + while (p < (buf + buf_len)) { + switch (*p) { + case '$': + if ((dollar && !lbrack) || !dollar) { + dollar = p; + } + p++; + continue; + case '{': + if (dollar && !lbrack) { + lbrack = p; + } + + p++; + continue; + case '}': + if (!(dollar && lbrack)) { + p++; + continue; + } + + // Store the leading bytes before ${ as a data field + tlen = dollar - s; + if (tlen > 0) { + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + _max_pkt_len += tlen; + } + + tmp = xcalloc(1, p - lbrack); + memcpy(tmp, lbrack + 1, p - lbrack - 1); + + if (udp_template_field_lookup(tmp, &c)) { + udp_template_add_field(t, c.ftype, c.length, + c.data); + _max_pkt_len += c.length; + // Push the pointer past the } if this was a + // valid variable + s = p + 1; + } else { + + // Rewind back to the ${ sequence if this was an + // invalid variable + s = dollar; + } + + free(tmp); + break; + default: + if (dollar && lbrack) { + p++; + continue; + } + } + dollar = NULL; + lbrack = NULL; + p++; + } + + // Store the trailing bytes as a final data field + if (s < p) { + tlen = p - s; + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + _max_pkt_len += tlen; + } + *max_pkt_len = _max_pkt_len; + return t; +} + +static fielddef_t fields[] = { + CLASSIFICATION_SUCCESS_FIELDSET_FIELDS, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + {.name = "udp_pkt_size", .type = "int", .desc = "UDP packet length"}, + {.name = "data", .type = "binary", .desc = "UDP payload"}, + ICMP_FIELDSET_FIELDS, +}; + +probe_module_t module_udp = { + .name = "udp", + .max_packet_length = 0, // set in init + .pcap_filter = "udp || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .thread_initialize = &udp_init_perthread, + .global_initialize = &udp_global_initialize, + .make_packet = &udp_make_packet, // can be overridden to udp_make_templated_packet by udp_global_initalize + .print_packet = &udp_print_packet, + .validate_packet = &udp_validate_packet, + .process_packet = &udp_process_packet, + .close = &udp_global_cleanup, + .helptext = "Probe module that sends UDP packets to hosts. Packets can " + "optionally be templated based on destination host. Specify " + "packet file with --probe-args=file:/path_to_packet_file " + "and templates with template:/path_to_template_file.", + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0])}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_udp.h b/rdns_scan/zmap4rdns/src/probe_modules/module_udp.h new file mode 100644 index 0000000..a0d6477 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_udp.h @@ -0,0 +1,95 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <string.h> + +#include <unistd.h> + +#include "aesrand.h" +#include "types.h" + +#include "state.h" + +#define NO_SRC_PORT_VALIDATION -1 + +typedef enum udp_payload_field_type { + UDP_DATA, + UDP_SADDR_N, + UDP_SADDR_A, + UDP_DADDR_N, + UDP_DADDR_A, + UDP_SPORT_N, + UDP_SPORT_A, + UDP_DPORT_N, + UDP_DPORT_A, + UDP_RAND_BYTE, + UDP_RAND_DIGIT, + UDP_RAND_ALPHA, + UDP_RAND_ALPHANUM +} udp_payload_field_type_t; + +typedef struct udp_payload_field_type_def { + const char *name; + const char *desc; + size_t max_length; + udp_payload_field_type_t ftype; +} udp_payload_field_type_def_t; + +typedef struct udp_payload_field { + enum udp_payload_field_type ftype; + size_t length; + char *data; +} udp_payload_field_t; + +typedef struct udp_payload_template { + unsigned int fcount; + struct udp_payload_field **fields; +} udp_payload_template_t; + +typedef struct udp_payload_output { + int length; + char *data; +} udp_payload_output_t; + +void udp_print_packet(FILE *fp, void *packet); + +int udp_make_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, uint8_t ttl, uint32_t *validation, + int probe_num, void *arg); +int udp_make_templated_packet(void *buf, size_t *buf_len, ipaddr_n_t src_ip, + ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, void *arg); + +int udp_do_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED uint32_t *src_ip, + uint32_t *validation, int num_ports, + int expected_port); + +void udp_set_num_ports(int); +int udp_global_initialize(struct state_conf *conf); +int udp_global_cleanup(UNUSED struct state_conf *zconf, + UNUSED struct state_send *zsend, + UNUSED struct state_recv *zrecv); + +void udp_template_add_field(udp_payload_template_t *t, + udp_payload_field_type_t ftype, unsigned int length, + char *data); + +void udp_template_free(udp_payload_template_t *t); + +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes); + +int udp_template_field_lookup(const char *vname, udp_payload_field_t *c); + +udp_payload_template_t *udp_template_load(uint8_t *buf, uint32_t buf_len, + uint32_t *max_pkt_len); diff --git a/rdns_scan/zmap4rdns/src/probe_modules/module_upnp.c b/rdns_scan/zmap4rdns/src/probe_modules/module_upnp.c new file mode 100644 index 0000000..3e336ba --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/module_upnp.c @@ -0,0 +1,274 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" +#include "module_udp.h" + +#define ICMP_UNREACH_HEADER_SIZE 8 + +static const char *upnp_query = "M-SEARCH * HTTP/1.1\r\n" + "Host:239.255.255.250:1900\r\n" + "ST:upnp:rootdevice\r\n" + "Man:\"ssdp:discover\"\r\nMX:3\r\n\r\n"; + +probe_module_t module_upnp; + +static int num_ports; + +int upnp_global_initialize(struct state_conf *state) +{ + num_ports = state->source_port_last - state->source_port_first + 1; + udp_set_num_ports(num_ports); + return EXIT_SUCCESS; +} + +int upnp_init_perthread(void *buf, macaddr_t *src, macaddr_t *gw, + port_h_t dst_port, + UNUSED void **arg_ptr) +{ + memset(buf, 0, MAX_PACKET_SIZE); + struct ether_header *eth_header = (struct ether_header *)buf; + make_eth_header(eth_header, src, gw); + struct ip *ip_header = (struct ip *)(ð_header[1]); + + uint16_t len = htons(sizeof(struct ip) + sizeof(struct udphdr) + + strlen(upnp_query)); + make_ip_header(ip_header, IPPROTO_UDP, len); + + struct udphdr *udp_header = (struct udphdr *)(&ip_header[1]); + len = sizeof(struct udphdr) + strlen(upnp_query); + make_udp_header(udp_header, dst_port, len); + + char *payload = (char *)(&udp_header[1]); + + assert(sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + strlen(upnp_query) <= + MAX_PACKET_SIZE); + + assert(MAX_PACKET_SIZE - ((char *)payload - (char *)buf) > + (int)strlen(upnp_query)); + strcpy(payload, upnp_query); + + return EXIT_SUCCESS; +} + +int upnp_validate_packet(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation) +{ + return udp_do_validate_packet(ip_hdr, len, src_ip, validation, + num_ports, zconf.target_port); +} + + +void upnp_process_packet(const u_char *packet, + UNUSED uint32_t len, fieldset_t *fs, + UNUSED uint32_t *validation, + UNUSED struct timespec ts) +{ + struct ip *ip_hdr = (struct ip *)&packet[sizeof(struct ether_header)]; + if (ip_hdr->ip_p == IPPROTO_UDP) { + struct udphdr *udp = + (struct udphdr *)((char *)ip_hdr + ip_hdr->ip_hl * 4); + + char *payload = (char *)(&udp[1]); + uint16_t plen = udp->uh_ulen - 8; + + char *s = xmalloc(plen + 1); + strncpy(s, payload, plen); + s[plen] = 0; + + int is_first = 1; + const char *classification = "none"; + uint64_t is_success = 0; + + char *server = NULL, *location = NULL, *usn = NULL, *st = NULL, + *cachecontrol = NULL, *ext = NULL, *xusragent = NULL, + *date = NULL, *agent = NULL; + + char *pch = strtok(s, "\n"); + while (pch != NULL) { + if (pch[strlen(pch) - 1] == '\r') { + pch[strlen(pch) - 1] = '\0'; + } + if (strlen(pch) == 0) { + pch = strtok(NULL, "\n"); + continue; + } + // the first pch is always supposed to be an HTTP + // response + if (is_first) { + if (strcmp(pch, "HTTP/1.1 200 OK")) { + classification = "no-http-header"; + is_success = 0; + goto cleanup; + } + is_first = 0; + is_success = 1; + classification = "upnp"; + pch = strtok(NULL, "\n"); + continue; + } + char *value = pch; + char *key = strsep(&value, ":"); + if (!key) { + pch = strtok(NULL, "\n"); + continue; + } + if (!value) { + pch = strtok(NULL, "\n"); + continue; + } + if (value[0] == ' ') { + value += (size_t)1; + } + if (!strcasecmp(key, "server")) { + server = strdup(value); + } else if (!strcasecmp(key, "location")) { + location = strdup(value); + } else if (!strcasecmp(key, "USN")) { + usn = strdup(value); + } else if (!strcasecmp(key, "EXT")) { + ext = strdup(value); + } else if (!strcasecmp(key, "ST")) { + st = strdup(value); + } else if (!strcasecmp(key, "Agent")) { + agent = strdup(value); + } else if (!strcasecmp(key, "X-User-Agent")) { + xusragent = strdup(value); + } else if (!strcasecmp(key, "date")) { + date = strdup(value); + } else if (!strcasecmp(key, "Cache-Control")) { + cachecontrol = strdup(value); + } else { + // log_debug("upnp-module", "new key: %s", key); + } + pch = strtok(NULL, "\n"); + } + + cleanup: + fs_add_string(fs, "classification", (char *)classification, 0); + fs_add_bool(fs, "success", is_success); + fs_chkadd_unsafe_string(fs, "server", server, 1); + fs_chkadd_unsafe_string(fs, "location", location, 1); + fs_chkadd_unsafe_string(fs, "usn", usn, 1); + fs_chkadd_unsafe_string(fs, "st", st, 1); + fs_chkadd_unsafe_string(fs, "ext", ext, 1); + fs_chkadd_unsafe_string(fs, "cache_control", cachecontrol, 1); + fs_chkadd_unsafe_string(fs, "x_user_agent", xusragent, 1); + fs_chkadd_unsafe_string(fs, "agent", agent, 1); + fs_chkadd_unsafe_string(fs, "date", date, 1); + fs_add_uint64(fs, "sport", ntohs(udp->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + + fs_add_binary(fs, "data", + (ntohs(udp->uh_ulen) - sizeof(struct udphdr)), + (void *)&udp[1], 0); + + free(s); + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + fs_add_constchar(fs, "classification", "icmp"); + fs_add_uint64(fs, "success", 0); + + fs_add_null(fs, "server"); + fs_add_null(fs, "location"); + fs_add_null(fs, "usn"); + fs_add_null(fs, "st"); + fs_add_null(fs, "ext"); + fs_add_null(fs, "cache_control"); + fs_add_null(fs, "x_user_agent"); + fs_add_null(fs, "agent"); + fs_add_null(fs, "date"); + + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + + fs_populate_icmp_from_iphdr(ip_hdr, len, fs); + fs_add_null(fs, "data"); + } else { + fs_add_constchar(fs, "classification", "other"); + fs_add_bool(fs, "success", 0); + fs_add_null(fs, "server"); + fs_add_null(fs, "location"); + fs_add_null(fs, "usn"); + fs_add_null(fs, "st"); + fs_add_null(fs, "ext"); + fs_add_null(fs, "cache_control"); + fs_add_null(fs, "x_user_agent"); + fs_add_null(fs, "agent"); + fs_add_null(fs, "date"); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "classification", + .type = "string", + .desc = "packet classification"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + + {.name = "server", .type = "string", .desc = "UPnP server"}, + {.name = "location", .type = "string", .desc = "UPnP location"}, + {.name = "usn", .type = "string", .desc = "UPnP usn"}, + {.name = "st", .type = "string", .desc = "UPnP st"}, + {.name = "ext", .type = "string", .desc = "UPnP ext"}, + {.name = "cache_control", .type = "string", .desc = "UPnP cache-control"}, + {.name = "x_user_agent", .type = "string", .desc = "UPnP x-user-agent"}, + {.name = "agent", .type = "string", .desc = "UPnP agent"}, + {.name = "date", .type = "string", .desc = "UPnP date"}, + + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + ICMP_FIELDSET_FIELDS, + {.name = "data", .type = "binary", .desc = "UDP payload"}}; + +probe_module_t module_upnp = { + .name = "upnp", + .max_packet_length = 139, + .pcap_filter = "udp || icmp", + .pcap_snaplen = 2048, + .port_args = 1, + .global_initialize = &upnp_global_initialize, + .thread_initialize = &upnp_init_perthread, + .make_packet = &udp_make_packet, + .print_packet = &udp_print_packet, + .process_packet = &upnp_process_packet, + .validate_packet = &upnp_validate_packet, + // UPnP isn't actually dynamic, however, we don't handle escaping + // properly in the CSV module and this will force users to use JSON. + .output_type = OUTPUT_TYPE_DYNAMIC, + .close = NULL, + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response.", + .fields = fields, + .numfields = 18}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/packet.c b/rdns_scan/zmap4rdns/src/probe_modules/packet.c new file mode 100644 index 0000000..c984049 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/packet.c @@ -0,0 +1,264 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <assert.h> + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "packet.h" + +#include "../state.h" + +#ifndef NDEBUG +void print_macaddr(struct ifreq *i) +{ + printf("Device %s -> Ethernet %02x:%02x:%02x:%02x:%02x:%02x\n", + i->ifr_name, (int)((unsigned char *)&i->ifr_addr.sa_data)[0], + (int)((unsigned char *)&i->ifr_addr.sa_data)[1], + (int)((unsigned char *)&i->ifr_addr.sa_data)[2], + (int)((unsigned char *)&i->ifr_addr.sa_data)[3], + (int)((unsigned char *)&i->ifr_addr.sa_data)[4], + (int)((unsigned char *)&i->ifr_addr.sa_data)[5]); +} +#endif /* NDEBUG */ + +#define IP_ADDR_LEN_STR 20 + +void fprintf_ip_header(FILE *fp, struct ip *iph) +{ + struct in_addr *s = (struct in_addr *)&(iph->ip_src); + struct in_addr *d = (struct in_addr *)&(iph->ip_dst); + + char srcip[IP_ADDR_LEN_STR + 1]; + char dstip[IP_ADDR_LEN_STR + 1]; + // inet_ntoa is a const char * so we if just call it in + // fprintf, you'll get back wrong results since we're + // calling it twice. + strncpy(srcip, inet_ntoa(*s), IP_ADDR_LEN_STR - 1); + strncpy(dstip, inet_ntoa(*d), IP_ADDR_LEN_STR - 1); + + srcip[IP_ADDR_LEN_STR] = '\0'; + dstip[IP_ADDR_LEN_STR] = '\0'; + + fprintf(fp, "ip { saddr: %s | daddr: %s | checksum: %#04X }\n", srcip, + dstip, ntohs(iph->ip_sum)); +} + +void fprintf_eth_header(FILE *fp, struct ether_header *ethh) +{ + if (!zconf.send_ip_pkts) { + fprintf(fp, + "eth { shost: %02x:%02x:%02x:%02x:%02x:%02x | " + "dhost: %02x:%02x:%02x:%02x:%02x:%02x }\n", + (int)((unsigned char *)ethh->ether_shost)[0], + (int)((unsigned char *)ethh->ether_shost)[1], + (int)((unsigned char *)ethh->ether_shost)[2], + (int)((unsigned char *)ethh->ether_shost)[3], + (int)((unsigned char *)ethh->ether_shost)[4], + (int)((unsigned char *)ethh->ether_shost)[5], + (int)((unsigned char *)ethh->ether_dhost)[0], + (int)((unsigned char *)ethh->ether_dhost)[1], + (int)((unsigned char *)ethh->ether_dhost)[2], + (int)((unsigned char *)ethh->ether_dhost)[3], + (int)((unsigned char *)ethh->ether_dhost)[4], + (int)((unsigned char *)ethh->ether_dhost)[5]); + } +} + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, macaddr_t *dst) +{ + memcpy(ethh->ether_shost, src, ETHER_ADDR_LEN); + memcpy(ethh->ether_dhost, dst, ETHER_ADDR_LEN); + ethh->ether_type = htons(ETHERTYPE_IP); +} + +void make_ip_header(struct ip *iph, uint8_t protocol, uint16_t len) +{ + iph->ip_hl = 5; // Internet Header Length + iph->ip_v = 4; // IPv4 + iph->ip_tos = 0; // Type of Service + iph->ip_len = len; + iph->ip_id = htons(54321); // identification number + iph->ip_off = 0; // fragmentation flag + iph->ip_ttl = MAXTTL; // time to live (TTL) + iph->ip_p = protocol; // upper layer protocol => TCP + // we set the checksum = 0 for now because that's + // what it needs to be when we run the IP checksum + iph->ip_sum = 0; +} + +void make_icmp_header(struct icmp *buf) +{ + memset(buf, 0, sizeof(struct icmp)); + buf->icmp_type = ICMP_ECHO; + buf->icmp_code = 0; + buf->icmp_seq = 0; +} + +void make_tcp_header(struct tcphdr *tcp_header, port_h_t dest_port, + uint16_t th_flags) +{ + tcp_header->th_seq = random(); + tcp_header->th_ack = 0; + tcp_header->th_x2 = 0; + tcp_header->th_off = 5; // data offset + tcp_header->th_flags = 0; + tcp_header->th_flags |= th_flags; + tcp_header->th_win = htons(65535); // largest possible window + tcp_header->th_sum = 0; + tcp_header->th_urp = 0; + tcp_header->th_dport = htons(dest_port); +} + +size_t set_mss_option(struct tcphdr *tcp_header) { + // This only sets MSS, which is a single-word option. + size_t header_size = tcp_header->th_off * 4; + uint8_t *base = (uint8_t *) tcp_header; + uint8_t *last_opt = (uint8_t*) base + header_size; + + // TCP Option "header" + last_opt[0] = 2; // MSS + last_opt[1] = 4; // MSS is 4 bytes long + + // Default Linux MSS is 1460, which 0x05b4 + last_opt[2] = 0x05; + last_opt[3] = 0xb4; + + tcp_header->th_off += 1; + return tcp_header->th_off*4; +} + +void make_udp_header(struct udphdr *udp_header, port_h_t dest_port, + uint16_t len) +{ + udp_header->uh_dport = htons(dest_port); + udp_header->uh_ulen = htons(len); + // checksum ignored in IPv4 if 0 + udp_header->uh_sum = 0; +} + +int icmp_helper_validate(const struct ip *ip_hdr, uint32_t len, + size_t min_l4_len, struct ip **probe_pkt, + size_t *probe_len) +{ + // We're only equipped to handle ICMP packets at this point + assert(ip_hdr->ip_p == IPPROTO_ICMP); + + // Several ICMP responses can be generated by hosts along the way in + // response to a non-ICMP probe packet. These include: + // * Source quench (ICMP_SOURCE_QUENCH) + // * Destination Unreachable (ICMP_DEST_UNREACH) + // * Redirect (ICMP_REDIRECT) + // * Time exceeded (ICMP_TIME_EXCEEDED) + // In all of these cases, the IP header and first 8 bytes of the + // original packet are included in the responses and can be used + // to understand where the probe packet was sent. + + // Check if the response was large enough to contain an IP header + const uint32_t min_len = 4 * ip_hdr->ip_hl + ICMP_HEADER_SIZE + + sizeof(struct ip) + min_l4_len; + if (len < min_len) { + return PACKET_INVALID; + } + // Check that ICMP response is one of these four + struct icmp *icmp = (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); + if (!(icmp->icmp_type == ICMP_UNREACH || + icmp->icmp_type == ICMP_SOURCEQUENCH || + icmp->icmp_type == ICMP_REDIRECT || + icmp->icmp_type == ICMP_TIMXCEED)) { + return PACKET_INVALID; + } + struct ip *ip_inner = (struct ip *)((char *)icmp + ICMP_HEADER_SIZE); + size_t inner_packet_len = len - (4 * ip_hdr->ip_hl + ICMP_HEADER_SIZE); + // Now we know the actual inner ip length, we should recheck the buffer + // to make sure it has enough room for the application layer packet + if (inner_packet_len < (4 * ip_inner->ip_hl + min_l4_len)) { + return PACKET_INVALID; + } + // find original destination IP and check that we sent a packet + // to that IP address + uint32_t dest = ip_inner->ip_dst.s_addr; + if (!blocklist_is_allowed(dest)) { + return PACKET_INVALID; + } + *probe_pkt = ip_inner; + *probe_len = inner_packet_len; + return PACKET_VALID; +} + +void fs_add_null_icmp(fieldset_t *fs) +{ + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); +} + +void fs_add_failure_no_port(fieldset_t *fs) +{ + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_unreach_str"); +} + +void fs_populate_icmp_from_iphdr(struct ip *ip, size_t len, fieldset_t *fs) +{ + assert(ip && "no ip header provide to fs_populate_icmp_from_iphdr"); + assert(fs && "no fieldset provided to fs_populate_icmp_from_iphdr"); + struct icmp *icmp = get_icmp_header(ip, len); + assert(icmp); + // ICMP unreach comes from another server (not the one we sent a + // probe to); But we will fix up saddr to be who we sent the + // probe to, in case you care. + struct ip *ip_inner = get_inner_ip_header(icmp, len); + fs_modify_string(fs, "saddr", make_ip_str(ip_inner->ip_dst.s_addr), 1); + // Add other ICMP fields from within the header + fs_add_string(fs, "icmp_responder", make_ip_str(ip->ip_src.s_addr), 1); + fs_add_uint64(fs, "icmp_type", icmp->icmp_type); + fs_add_uint64(fs, "icmp_code", icmp->icmp_code); + if (icmp->icmp_code <= ICMP_UNREACH_PRECEDENCE_CUTOFF) { + fs_add_constchar(fs, "icmp_unreach_str", + icmp_unreach_strings[icmp->icmp_code]); + } else { + fs_add_constchar(fs, "icmp_unreach_str", "unknown"); + } +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip) +{ + struct in_addr t; + t.s_addr = ip; + const char *temp = inet_ntoa(t); + char *retv = xmalloc(strlen(temp) + 1); + strcpy(retv, temp); + return retv; +} + +const char *icmp_unreach_strings[] = { + "network unreachable", + "host unreachable", + "protocol unreachable", + "port unreachable", + "fragments required", + "source route failed", + "network unknown", + "host unknown", + "source host isolated", + "network admin. prohibited", + "host admin. prohibited", + "network unreachable TOS", + "host unreachable TOS", + "communication admin. prohibited", + "host presdence violation", + "precedence cutoff" +}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/packet.h b/rdns_scan/zmap4rdns/src/probe_modules/packet.h new file mode 100644 index 0000000..5876a46 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/packet.h @@ -0,0 +1,235 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../../lib/includes.h" +#include "../../lib/blocklist.h" +#include "../state.h" + +#ifndef PACKET_H +#define PACKET_H + +#define MAX_PACKET_SIZE 4096 + +#define ICMP_UNREACH_HEADER_SIZE 8 + +#define PACKET_VALID 1 +#define PACKET_INVALID 0 + +#define ICMP_HEADER_SIZE 8 + +#define PRINT_PACKET_SEP \ + "------------------------------------------------------\n" + +#define CLASSIFICATION_SUCCESS_FIELDSET_FIELDS \ + {.name = "classification", \ + .type = "string", \ + .desc = "packet classification"}, \ + {.name = "success", \ + .type = "bool", \ + .desc = "is response considered success"} + +#define CLASSIFICATION_SUCCESS_FIELDSET_LEN 2 + +#define ICMP_FIELDSET_FIELDS \ + {.name = "icmp_responder", \ + .type = "string", \ + .desc = "Source IP of ICMP_UNREACH messages"}, \ + {.name = "icmp_type", .type = "int", .desc = "icmp message type"}, \ + {.name = "icmp_code", .type = "int", .desc = "icmp message sub type code"}, \ + {.name = "icmp_unreach_str", \ + .type = "string", \ + .desc = "for icmp_unreach responses, the string version of icmp_code (e.g. network-unreach)"} + +#define ICMP_FIELDSET_LEN 4 + + +typedef unsigned short __attribute__((__may_alias__)) alias_unsigned_short; + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, macaddr_t *dst); + +void make_ip_header(struct ip *iph, uint8_t, uint16_t); +void make_tcp_header(struct tcphdr *, port_h_t, uint16_t); +size_t set_mss_option(struct tcphdr *tcp_header); +void make_icmp_header(struct icmp *); +void make_udp_header(struct udphdr *udp_header, port_h_t dest_port, + uint16_t len); +void fprintf_ip_header(FILE *fp, struct ip *iph); +void fprintf_eth_header(FILE *fp, struct ether_header *ethh); + +static inline unsigned short in_checksum(unsigned short *ip_pkt, int len) +{ + unsigned long sum = 0; + for (int nwords = len / 2; nwords > 0; nwords--) { + sum += *ip_pkt++; + } + if (len % 2 == 1) { + sum += *((unsigned char *) ip_pkt); + } + sum = (sum >> 16) + (sum & 0xffff); + return (unsigned short)(~sum); +} + +static inline unsigned short in_icmp_checksum(unsigned short *ip_pkt, int len) +{ + unsigned long sum = 0; + for (int nwords = len / 2; nwords > 0; nwords--) { + sum += *ip_pkt++; + } + if (len % 2 == 1) { + sum += *((unsigned char *) ip_pkt); + } + sum = (sum >> 16) + (sum & 0xffff); + return (unsigned short)(~sum); +} + +static inline unsigned short +zmap_ip_checksum(unsigned short *buf) +{ + return in_checksum(buf, (int)sizeof(struct ip)); +} + + +static inline unsigned short +icmp_checksum(unsigned short *buf, size_t buflen) +{ + return in_icmp_checksum(buf, buflen); +} + +static inline uint16_t tcp_checksum(unsigned short len_tcp, + uint32_t saddr, + uint32_t daddr, + struct tcphdr *tcp_pkt) +{ + alias_unsigned_short *src_addr = (alias_unsigned_short *)&saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *)&daddr; + + unsigned char prot_tcp = 6; + unsigned long sum = 0; + int nleft = len_tcp; + unsigned short *w; + + w = (unsigned short *)tcp_pkt; + // calculate the checksum for the tcp header and tcp data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + // if nleft is 1 there is still one byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & ntohs(0xFF00); + } + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += htons(len_tcp); + sum += htons(prot_tcp); + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + // Take the one's complement of sum + return (unsigned short)(~sum); +} + +// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise +static inline int +check_dst_port(uint16_t port, int num_ports, uint32_t *validation) +{ + if (port > zconf.source_port_last || port < zconf.source_port_first) { + return 0; + } + int32_t to_validate = port - zconf.source_port_first; + int32_t min = validation[1] % num_ports; + int32_t max = (validation[1] + zconf.packet_streams - 1) % num_ports; + + return (((max - min) % num_ports) >= ((to_validate - min) % num_ports)); +} + +static inline uint16_t +get_src_port(int num_ports, int probe_num, uint32_t *validation) +{ + return zconf.source_port_first + + ((validation[1] + probe_num) % num_ports); +} + +static inline struct ip *get_ip_header(const u_char *packet, uint32_t len) +{ + if (len < sizeof(struct ether_header)) { + return NULL; + } + return (struct ip *)&packet[sizeof(struct ether_header)]; +} + +static inline struct tcphdr *get_tcp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct tcphdr)) > len) { + return NULL; + } + return (struct tcphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline struct udphdr *get_udp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct udphdr)) > len) { + return NULL; + } + return (struct udphdr *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline struct icmp *get_icmp_header(const struct ip *ip_hdr, + uint32_t len) +{ + // buf not large enough to contain expected udp header + if ((4 * ip_hdr->ip_hl + sizeof(struct icmp)) > len) { + return NULL; + } + return (struct icmp *)((char *)ip_hdr + 4 * ip_hdr->ip_hl); +} + +static inline uint8_t *get_udp_payload(const struct udphdr *udp, + UNUSED uint32_t len) +{ + return (uint8_t*)(&udp[1]); +} + +static inline struct ip *get_inner_ip_header(const struct icmp *icmp, + uint32_t len) +{ + if (len < (ICMP_UNREACH_HEADER_SIZE + sizeof(struct ip))) { + return NULL; + } + return (struct ip *)((char *)icmp + ICMP_UNREACH_HEADER_SIZE); +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip); + +extern const char *icmp_unreach_strings[]; + +int icmp_helper_validate(const struct ip *ip_hdr, uint32_t len, + size_t min_l4_len, struct ip **probe_pkt, + size_t *probe_len); + +void fs_add_null_icmp(fieldset_t *fs); + +void fs_populate_icmp_from_iphdr(struct ip *ip, size_t len, fieldset_t *fs); + +#endif diff --git a/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.c b/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.c new file mode 100644 index 0000000..f1027bb --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.c @@ -0,0 +1,128 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdio.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <sys/time.h> + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "probe_modules.h" +#include "packet.h" + +extern probe_module_t module_tcp_synscan; +extern probe_module_t module_tcp_synackscan; +extern probe_module_t module_icmp_echo; +extern probe_module_t module_icmp_echo_time; +extern probe_module_t module_udp; +extern probe_module_t module_ntp; +extern probe_module_t module_upnp; +extern probe_module_t module_dns; +extern probe_module_t module_bacnet; +// ADD YOUR MODULE HERE + +probe_module_t *probe_modules[] = { + &module_tcp_synscan, &module_tcp_synackscan, &module_icmp_echo, + &module_icmp_echo_time, &module_udp, &module_ntp, &module_upnp, &module_dns, + &module_bacnet + // ADD YOUR MODULE HERE +}; + +probe_module_t *get_probe_module_by_name(const char *name) +{ + int len = (int)(sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + if (!strcmp(probe_modules[i]->name, name)) { + return probe_modules[i]; + } + } + return NULL; +} + +void print_probe_modules(void) +{ + int len = (int)(sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + printf("%s\n", probe_modules[i]->name); + } +} + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip) +{ + // WARNING: you must update fs_ip_fields_len as well + // as the definitions set (ip_fiels) if you + // change the fields added below: + fs_add_string(fs, "saddr", make_ip_str(ip->ip_src.s_addr), 1); + fs_add_uint64(fs, "saddr_raw", (uint64_t)ip->ip_src.s_addr); + fs_add_string(fs, "daddr", make_ip_str(ip->ip_dst.s_addr), 1); + fs_add_uint64(fs, "daddr_raw", (uint64_t)ip->ip_dst.s_addr); + fs_add_uint64(fs, "ipid", ntohs(ip->ip_id)); + fs_add_uint64(fs, "ttl", ip->ip_ttl); +} + +#define TIMESTR_LEN 55 + +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) +{ + fs_add_bool(fs, "repeat", is_repeat); + fs_add_bool(fs, "cooldown", in_cooldown); + + char *timestr = xmalloc(TIMESTR_LEN + 1); + char *timestr_ms = xmalloc(TIMESTR_LEN + 1); + struct timeval t; + gettimeofday(&t, NULL); + struct tm *ptm = localtime(&t.tv_sec); + strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S.%%03d%z", ptm); + snprintf(timestr_ms, TIMESTR_LEN, timestr, t.tv_usec / 1000); + free(timestr); + fs_add_string(fs, "timestamp_str", timestr_ms, 1); + fs_add_uint64(fs, "timestamp_ts", (uint64_t)t.tv_sec); + fs_add_uint64(fs, "timestamp_us", (uint64_t)t.tv_usec); +} + +int ip_fields_len = 6; +fielddef_t ip_fields[] = { + {.name = "saddr", + .type = "string", + .desc = "source IP address of response"}, + {.name = "saddr_raw", + .type = "int", + .desc = "network order integer form of source IP address"}, + {.name = "daddr", + .type = "string", + .desc = "destination IP address of response"}, + {.name = "daddr_raw", + .type = "int", + .desc = "network order integer form of destination IP address"}, + {.name = "ipid", + .type = "int", + .desc = "IP identification number of response"}, + {.name = "ttl", .type = "int", .desc = "time-to-live of response packet"}}; + +int sys_fields_len = 5; +fielddef_t sys_fields[] = { + {.name = "repeat", + .type = "bool", + .desc = "Is response a repeat response from host"}, + {.name = "cooldown", + .type = "bool", + .desc = "Was response received during the cooldown period"}, + {.name = "timestamp_str", + .type = "string", + .desc = "timestamp of when response arrived in ISO8601 format."}, + {.name = "timestamp_ts", + .type = "int", + .desc = "timestamp of when response arrived in seconds since Epoch"}, + {.name = "timestamp_us", + .type = "int", + .desc = + "microsecond part of timestamp (e.g. microseconds since 'timestamp-ts')"}}; diff --git a/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.h b/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.h new file mode 100644 index 0000000..963b5a5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/probe_modules/probe_modules.h @@ -0,0 +1,104 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../state.h" +#include "../fieldset.h" + +#ifndef PROBE_MODULES_H +#define PROBE_MODULES_H + +#define OUTPUT_TYPE_STATIC 1 +#define OUTPUT_TYPE_DYNAMIC 2 + +typedef struct probe_response_type { + const uint8_t is_success; + const char *name; +} response_type_t; + +typedef int (*probe_global_init_cb)(struct state_conf *); +typedef int (*probe_thread_init_cb)(void *packetbuf, macaddr_t *src_mac, + macaddr_t *gw_mac, port_n_t src_port, + void **arg_ptr); + +// The make_packet callback is passed a buffer pointing at an ethernet header. +// The buffer is MAX_PACKET_SIZE bytes. The callback must update the value +// pointed at by buf_len with the actual length of the packet. The contents of +// the buffer will match the previous packet sent. Every invocation of +// make_packet contains a unique (src_ip, probe_num) tuple. +// +// The probe module is responsible for populating the IP header. The src_ip, +// dst_ip, and ttl are provided by the framework and must be set on the IP +// header. +// +// The uin32_t validation parameter is a pointer to four 4-byte words of +// validation data. The data is deterministic based on the the validation +// state, and is constant across a src_ip. To get the src_port, use the +// get_src_port function which takes probe_num and validation as parameters. +typedef int (*probe_make_packet_cb)(void *packetbuf, size_t *buf_len, + ipaddr_n_t src_ip, ipaddr_n_t dst_ip, uint8_t ttl, + uint32_t *validation, int probe_num, + void *arg); + +typedef void (*probe_print_packet_cb)(FILE *, void *packetbuf); +typedef int (*probe_close_cb)(struct state_conf *, struct state_send *, + struct state_recv *); +typedef int (*probe_validate_packet_cb)(const struct ip *ip_hdr, uint32_t len, + uint32_t *src_ip, uint32_t *validation); + +typedef void (*probe_classify_packet_cb)(const u_char *packetbuf, uint32_t len, + fieldset_t *, uint32_t *validation, const struct timespec ts); + +typedef struct probe_module { + const char *name; + + // TODO(dadrian): Completely get rid of this. We can do bandwidth rate + // limiting by actually counting how much data is sent over the wire. We + // know the lengths of packets from the make_packet API. + size_t max_packet_length; + + const char *pcap_filter; + size_t pcap_snaplen; + + // Should ZMap complain if the user hasn't specified valid + // source and target port numbers? + uint8_t port_args; + + probe_global_init_cb global_initialize; + probe_thread_init_cb thread_initialize; + probe_make_packet_cb make_packet; + probe_print_packet_cb print_packet; + probe_validate_packet_cb validate_packet; + probe_classify_packet_cb process_packet; + probe_close_cb close; + int output_type; + fielddef_t *fields; + int numfields; + const char *helptext; + +} probe_module_t; + +probe_module_t *get_probe_module_by_name(const char *); + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip); +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown); +void print_probe_modules(void); + +extern int ip_fields_len; +extern int sys_fields_len; +extern fielddef_t ip_fields[]; +extern fielddef_t sys_fields[]; + +#endif // HEADER_PROBE_MODULES_H diff --git a/rdns_scan/zmap4rdns/src/recv-internal.h b/rdns_scan/zmap4rdns/src/recv-internal.h new file mode 100644 index 0000000..1056110 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/recv-internal.h @@ -0,0 +1,19 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_RECV_INTERNAL_H +#define ZMAP_RECV_INTERNAL_H + +#include <stdint.h> + +void handle_packet(uint32_t buflen, const uint8_t *bytes, const struct timespec ts); +void recv_init(); +void recv_packets(); +void recv_cleanup(); + +#endif /* ZMAP_RECV_INTERNAL_H */ diff --git a/rdns_scan/zmap4rdns/src/recv-pcap.c b/rdns_scan/zmap4rdns/src/recv-pcap.c new file mode 100644 index 0000000..cc59cdd --- /dev/null +++ b/rdns_scan/zmap4rdns/src/recv-pcap.c @@ -0,0 +1,159 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <pthread.h> +#include <assert.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include <pcap.h> +#include <pcap/pcap.h> +#if defined __linux__ && __linux__ +#include <pcap/sll.h> +#endif + +#include "recv-internal.h" +#include "state.h" + +#include "probe_modules/probe_modules.h" + +#define PCAP_PROMISC 1 +#define PCAP_TIMEOUT 1000 + +static pcap_t *pc = NULL; + +void packet_cb(u_char __attribute__((__unused__)) * user, + const struct pcap_pkthdr *p, const u_char *bytes) +{ + struct timespec ts; + if (!p) { + return; + } + if (zrecv.filter_success >= zconf.max_results) { + // Libpcap can process multiple packets per pcap_dispatch; + // we need to throw out results once we've + // gotten our --max-results worth. + return; + } + + // length of entire packet captured by libpcap + uint32_t buflen = (uint32_t)p->caplen; + ts.tv_sec = p->ts.tv_sec; + ts.tv_nsec = p->ts.tv_usec * 1000; + handle_packet(buflen, bytes, ts); +} + +#define BPFLEN 1024 + +void recv_init() +{ + char bpftmp[BPFLEN]; + char errbuf[PCAP_ERRBUF_SIZE]; + + pc = pcap_open_live(zconf.iface, zconf.probe_module->pcap_snaplen, + PCAP_PROMISC, PCAP_TIMEOUT, errbuf); + if (pc == NULL) { + log_fatal("recv", "could not open device %s: %s", zconf.iface, + errbuf); + } + switch (pcap_datalink(pc)) { + case DLT_EN10MB: + log_debug("recv", "Data link layer Ethernet"); + zconf.data_link_size = sizeof(struct ether_header); + break; + case DLT_RAW: + log_info("recv", "Data link RAW"); + zconf.data_link_size = 0; + break; +#if defined __linux__ && __linux__ + case DLT_LINUX_SLL: + log_info("recv", "Data link cooked socket"); + zconf.data_link_size = SLL_HDR_LEN; + break; +#endif + default: + log_error("recv", "unknown data link layer"); + } + + struct bpf_program bpf; + + if (!zconf.send_ip_pkts) { + snprintf(bpftmp, sizeof(bpftmp) - 1, + "not ether src %02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + assert(strlen(zconf.probe_module->pcap_filter) + 10 < + (BPFLEN - strlen(bpftmp))); + } else { + bpftmp[0] = 0; + } + if (zconf.probe_module->pcap_filter) { + if (!zconf.send_ip_pkts) { + strcat(bpftmp, " and ("); + } else { + strcat(bpftmp, "("); + } + strcat(bpftmp, zconf.probe_module->pcap_filter); + strcat(bpftmp, ")"); + } + if (strcmp(bpftmp, "")) { + if (pcap_compile(pc, &bpf, bpftmp, 1, 0) < 0) { + log_fatal("recv", "couldn't compile filter"); + } + if (pcap_setfilter(pc, &bpf) < 0) { + log_fatal("recv", "couldn't install filter"); + } + } + // set pcap_dispatch to not hang if it never receives any packets + // this could occur if you ever scan a small number of hosts as + // documented in issue #74. + if (pcap_setnonblock(pc, 1, errbuf) == -1) { + log_fatal("recv", "pcap_setnonblock error:%s", errbuf); + } +} + +void recv_packets() +{ + int ret = pcap_dispatch(pc, -1, packet_cb, NULL); + if (ret == -1) { + log_fatal("recv", "pcap_dispatch error"); + } else if (ret == 0) { + usleep(1000); + } +} + +void recv_cleanup() +{ + pcap_close(pc); + pc = NULL; +} + +int recv_update_stats(void) +{ + if (!pc) { + return EXIT_FAILURE; + } + struct pcap_stat pcst; + if (pcap_stats(pc, &pcst)) { + log_error("recv", "unable to retrieve pcap statistics: %s", + pcap_geterr(pc)); + return EXIT_FAILURE; + } else { + zrecv.pcap_recv = pcst.ps_recv; + zrecv.pcap_drop = pcst.ps_drop; + zrecv.pcap_ifdrop = pcst.ps_ifdrop; + } + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/recv-pfring.c b/rdns_scan/zmap4rdns/src/recv-pfring.c new file mode 100644 index 0000000..dbc4742 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/recv-pfring.c @@ -0,0 +1,82 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" +#include "recv-internal.h" + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include <errno.h> +#include <unistd.h> + +#include <pfring_zc.h> + +#include "state.h" + +static pfring_zc_pkt_buff *pf_buffer; +static pfring_zc_queue *pf_recv; + +void recv_init() +{ + // Get the socket and packet handle + pf_recv = zconf.pf.recv; + pf_buffer = pfring_zc_get_packet_handle(zconf.pf.cluster); + if (pf_buffer == NULL) { + log_fatal("recv", "Could not get packet handle: %s", + strerror(errno)); + } +} + +void recv_cleanup() +{ + if (!pf_recv) { + return; + } + pfring_zc_sync_queue(pf_recv, rx_only); +} + +void recv_packets() +{ + int ret; + // Poll for packets + do { + ret = pfring_zc_recv_pkt(pf_recv, &pf_buffer, 0); + if (ret == 0) { + usleep(1000); + } + } while (ret == 0); + // Handle other errors, by not doing anything and logging + if (ret != 1) { + log_error("recv", "Error: %d", ret); + return; + } + // Successfully got a packet, now handle it + struct timespec ts; + ts.tv_sec = pf_buffer->ts.tv_sec; + ts.tv_nsec = pf_buffer->ts.tv_nsec; //* 1000; + + uint8_t *pkt_buf = pfring_zc_pkt_buff_data(pf_buffer, pf_recv); + handle_packet(pf_buffer->len, pkt_buf, ts); +} + +int recv_update_stats(void) +{ + if (!pf_recv) { + return EXIT_FAILURE; + } + pfring_zc_stat pfst; + if (pfring_zc_stats(pf_recv, &pfst)) { + log_error("recv", "unable to retrieve pfring statistics"); + return EXIT_FAILURE; + } else { + zrecv.pcap_recv = pfst.recv; + zrecv.pcap_drop = pfst.drop; + } + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/recv.c b/rdns_scan/zmap4rdns/src/recv.c new file mode 100644 index 0000000..300a154 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/recv.c @@ -0,0 +1,198 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" + +#include <assert.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/pbm.h" + +#include <pthread.h> +#include <unistd.h> + +#include "recv-internal.h" +#include "state.h" +#include "validate.h" +#include "fieldset.h" +#include "expression.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/output_modules.h" + +static u_char fake_eth_hdr[65535]; +// bitmap of observed IP addresses +static uint8_t **seen = NULL; + +void handle_packet(uint32_t buflen, const u_char *bytes, + const struct timespec ts) +{ + if ((sizeof(struct ip) + zconf.data_link_size) > buflen) { + // buffer not large enough to contain ethernet + // and ip headers. further action would overrun buf + return; + } + struct ip *ip_hdr = (struct ip *)&bytes[zconf.data_link_size]; + + uint32_t src_ip = ip_hdr->ip_src.s_addr; + + uint32_t validation[VALIDATE_BYTES / sizeof(uint8_t)]; + // TODO: for TTL exceeded messages, ip_hdr->saddr is going to be + // different and we must calculate off potential payload message instead + validate_gen(ip_hdr->ip_dst.s_addr, ip_hdr->ip_src.s_addr, + (uint8_t *)validation); + + if (!zconf.probe_module->validate_packet( + ip_hdr, + buflen - (zconf.send_ip_pkts ? 0 : sizeof(struct ether_header)), + &src_ip, validation)) { + zrecv.validation_failed++; + return; + } else { + zrecv.validation_passed++; + } + // woo! We've validated that the packet is a response to our scan + int is_repeat = pbm_check(seen, ntohl(src_ip)); + // track whether this is the first packet in an IP fragment. + if (ip_hdr->ip_off & IP_MF) { + zrecv.ip_fragments++; + } + + fieldset_t *fs = fs_new_fieldset(&zconf.fsconf.defs); + fs_add_ip_fields(fs, ip_hdr); + // HACK: + // probe modules expect the full ethernet frame + // in process_packet. For VPN, we only get back an IP frame. + // Here, we fake an ethernet frame (which is initialized to + // have ETH_P_IP proto and 00s for dest/src). + if (zconf.send_ip_pkts) { + if (buflen > sizeof(fake_eth_hdr)) { + buflen = sizeof(fake_eth_hdr); + } + memcpy(&fake_eth_hdr[sizeof(struct ether_header)], + bytes + zconf.data_link_size, buflen); + bytes = fake_eth_hdr; + } + zconf.probe_module->process_packet(bytes, buflen, fs, validation, ts); + fs_add_system_fields(fs, is_repeat, zsend.complete); + int success_index = zconf.fsconf.success_index; + assert(success_index < fs->len); + int is_success = fs_get_uint64_by_index(fs, success_index); + + if (is_success) { + zrecv.success_total++; + if (!is_repeat) { + zrecv.success_unique++; + pbm_set(seen, ntohl(src_ip)); + } + if (zsend.complete) { + zrecv.cooldown_total++; + if (!is_repeat) { + zrecv.cooldown_unique++; + } + } + } else { + zrecv.failure_total++; + } + // probe module includes app_success field + if (zconf.fsconf.app_success_index >= 0) { + int is_app_success = + fs_get_uint64_by_index(fs, zconf.fsconf.app_success_index); + if (is_app_success) { + zrecv.app_success_total++; + if (!is_repeat) { + zrecv.app_success_unique++; + } + } + } + + fieldset_t *o = NULL; + // we need to translate the data provided by the probe module + // into a fieldset that can be used by the output module + if (!is_success && zconf.default_mode) { + goto cleanup; + } + if (is_repeat && zconf.default_mode) { + goto cleanup; + } + if (!evaluate_expression(zconf.filter.expression, fs)) { + goto cleanup; + } + zrecv.filter_success++; + o = translate_fieldset(fs, &zconf.fsconf.translation); + if (zconf.output_module && zconf.output_module->process_ip) { + zconf.output_module->process_ip(o); + } +cleanup: + fs_free(fs); + free(o); + if (zconf.output_module && zconf.output_module->update && + !(zrecv.success_unique % zconf.output_module->update_interval)) { + zconf.output_module->update(&zconf, &zsend, &zrecv); + } +} + +int recv_run(pthread_mutex_t *recv_ready_mutex) +{ + log_trace("recv", "recv thread started"); + log_debug("recv", "capturing responses on %s", zconf.iface); + if (!zconf.dryrun) { + recv_init(); + } + if (zconf.send_ip_pkts) { + struct ether_header *eth = (struct ether_header *)fake_eth_hdr; + memset(fake_eth_hdr, 0, sizeof(fake_eth_hdr)); + eth->ether_type = htons(ETHERTYPE_IP); + } + // initialize paged bitmap + seen = pbm_init(); + if (zconf.default_mode) { + log_info("recv", + "duplicate responses will be excluded from output"); + log_info("recv", + "unsuccessful responses will be excluded from output"); + } else { + log_info( + "recv", + "duplicate responses will be passed to the output module"); + log_info( + "recv", + "unsuccessful responses will be passed to the output module"); + } + pthread_mutex_lock(recv_ready_mutex); + zconf.recv_ready = 1; + pthread_mutex_unlock(recv_ready_mutex); + zrecv.start = now(); + if (zconf.max_results == 0) { + zconf.max_results = -1; + } + + do { + if (zconf.dryrun) { + sleep(1); + } else { + recv_packets(); + if (zconf.max_results && + zrecv.filter_success >= zconf.max_results) { + break; + } + } + } while ( + !(zsend.complete && (now() - zsend.finish > zconf.cooldown_secs))); + zrecv.finish = now(); + // get final pcap statistics before closing + recv_update_stats(); + if (!zconf.dryrun) { + pthread_mutex_lock(recv_ready_mutex); + recv_cleanup(); + pthread_mutex_unlock(recv_ready_mutex); + } + zrecv.complete = 1; + log_debug("recv", "thread finished"); + return 0; +} diff --git a/rdns_scan/zmap4rdns/src/recv.h b/rdns_scan/zmap4rdns/src/recv.h new file mode 100644 index 0000000..07824f0 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/recv.h @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_RECV_H +#define ZMAP_RECV_H + +#include <pthread.h> + +int recv_update_stats(void); +int recv_run(pthread_mutex_t *recv_ready_mutex); + +#endif /* ZMP_RECV_H */ diff --git a/rdns_scan/zmap4rdns/src/send-bsd.h b/rdns_scan/zmap4rdns/src/send-bsd.h new file mode 100644 index 0000000..249888a --- /dev/null +++ b/rdns_scan/zmap4rdns/src/send-bsd.h @@ -0,0 +1,37 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SEND_BSD_H +#define ZMAP_SEND_BSD_H + +#include <sys/types.h> +#include <sys/time.h> +#include <sys/ioctl.h> +#include <fcntl.h> + +#include "../lib/includes.h" + +#include <netinet/in.h> +#include <net/bpf.h> + +#ifdef ZMAP_SEND_LINUX_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +int send_run_init(UNUSED sock_t sock) +{ + // Don't need to do anything on BSD-like variants + return EXIT_SUCCESS; +} + +int send_packet(sock_t sock, void *buf, int len, UNUSED uint32_t idx) +{ + return write(sock.sock, buf, len); +} + +#endif /* ZMAP_SEND_BSD_H */ diff --git a/rdns_scan/zmap4rdns/src/send-linux.h b/rdns_scan/zmap4rdns/src/send-linux.h new file mode 100644 index 0000000..88866d5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/send-linux.h @@ -0,0 +1,60 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SEND_LINUX_H +#define ZMAP_SEND_LINUX_H + +#include "../lib/includes.h" +#include <sys/ioctl.h> + +#include <netpacket/packet.h> + +#ifdef ZMAP_SEND_BSD_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +// Dummy sockaddr for sendto +static struct sockaddr_ll sockaddr; + +int send_run_init(sock_t s) +{ + // Get the actual socket + int sock = s.sock; + // get source interface index + struct ifreq if_idx; + memset(&if_idx, 0, sizeof(struct ifreq)); + if (strlen(zconf.iface) >= IFNAMSIZ) { + log_error("send", "device interface name (%s) too long\n", + zconf.iface); + return EXIT_FAILURE; + } + strncpy(if_idx.ifr_name, zconf.iface, IFNAMSIZ - 1); + if (ioctl(sock, SIOCGIFINDEX, &if_idx) < 0) { + perror("SIOCGIFINDEX"); + return EXIT_FAILURE; + } + int ifindex = if_idx.ifr_ifindex; + + // destination address for the socket + memset((void *)&sockaddr, 0, sizeof(struct sockaddr_ll)); + sockaddr.sll_ifindex = ifindex; + sockaddr.sll_halen = ETH_ALEN; + if (zconf.send_ip_pkts) { + sockaddr.sll_protocol = htons(ETHERTYPE_IP); + } + memcpy(sockaddr.sll_addr, zconf.gw_mac, ETH_ALEN); + return EXIT_SUCCESS; +} + +int send_packet(sock_t sock, void *buf, int len, UNUSED uint32_t idx) +{ + return sendto(sock.sock, buf, len, 0, (struct sockaddr *)&sockaddr, + sizeof(struct sockaddr_ll)); +} + +#endif /* ZMAP_SEND_LINUX_H */ diff --git a/rdns_scan/zmap4rdns/src/send-pfring.h b/rdns_scan/zmap4rdns/src/send-pfring.h new file mode 100644 index 0000000..9ef29b2 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/send-pfring.h @@ -0,0 +1,42 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SEND_PFRING_H +#define ZMAP_SEND_PFRING_H + +#include "../lib/includes.h" +#include <sys/ioctl.h> + +#if defined(ZMAP_SEND_BSD_H) || defined(ZMAP_SEND_LINUX_H) +#error "Don't include send-bsd.h or send-linux.h with send-pfring.h" +#endif + +int send_run_init(sock_t socket) +{ + (void)socket; + + // All init for pfring happens in get_socket + return 0; +} + +int send_packet(sock_t sock, void *buf, int len, uint32_t idx) +{ + sock.pf.buffers[idx]->len = len; + memcpy(pfring_zc_pkt_buff_data(sock.pf.buffers[idx], sock.pf.queue), + buf, len); + int ret; + do { + ret = + pfring_zc_send_pkt(sock.pf.queue, &sock.pf.buffers[idx], 0); + } while (ret < 0); + return ret; +} + +void send_finish(sock_t sock) { pfring_zc_sync_queue(sock.pf.queue, tx_only); } + +#endif /* ZMAP_SEND_PFRING_H */ diff --git a/rdns_scan/zmap4rdns/src/send.c b/rdns_scan/zmap4rdns/src/send.c new file mode 100644 index 0000000..d8f4690 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/send.c @@ -0,0 +1,477 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "send.h" + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <time.h> +#include <unistd.h> +#include <pthread.h> +#include <string.h> +#include <errno.h> +#include <assert.h> +#include <signal.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/blocklist.h" +#include "../lib/lockfd.h" +#include "../lib/pbm.h" + +#include "aesrand.h" +#include "get_gateway.h" +#include "iterator.h" +#include "probe_modules/packet.h" +#include "probe_modules/probe_modules.h" +#include "shard.h" +#include "state.h" +#include "validate.h" + +// OS specific functions called by send_run +static inline int send_packet(sock_t sock, void *buf, int len, uint32_t idx); +static inline int send_run_init(sock_t sock); + +// Include the right implementations +#if defined(PFRING) +#include "send-pfring.h" +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#include "send-bsd.h" +#else /* LINUX */ +#include "send-linux.h" +#endif /* __APPLE__ || __FreeBSD__ || __NetBSD__ || __DragonFly__ */ + +// The iterator over the cyclic group + +// Lock for send run +static pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER; + +// Source ports for outgoing packets +static uint16_t num_src_ports; + +void sig_handler_increase_speed(UNUSED int signal) +{ + int old_rate = zconf.rate; + zconf.rate += (zconf.rate * 0.05); + log_info("send", "send rate increased from %i to %i pps.", old_rate, + zconf.rate); +} + +void sig_handler_decrease_speed(UNUSED int signal) +{ + int old_rate = zconf.rate; + zconf.rate -= (zconf.rate * 0.05); + log_info("send", "send rate decreased from %i to %i pps.", old_rate, + zconf.rate); +} + +// global sender initialize (not thread specific) +iterator_t *send_init(void) +{ + // generate a new primitive root and starting position + iterator_t *it; + uint32_t num_subshards = + (uint32_t)zconf.senders * (uint32_t)zconf.total_shards; + if (num_subshards > blocklist_count_allowed()) { + log_fatal("send", "senders * shards > allowed probes"); + } + if (zsend.max_targets && (num_subshards > zsend.max_targets)) { + log_fatal("send", "senders * shards > max targets"); + } + it = iterator_init(zconf.senders, zconf.shard_num, zconf.total_shards); + // determine the source address offset from which we'll send packets + struct in_addr temp; + temp.s_addr = zconf.source_ip_addresses[0]; + log_debug("send", "srcip_first: %s", inet_ntoa(temp)); + temp.s_addr = zconf.source_ip_addresses[zconf.number_source_ips - 1]; + log_debug("send", "srcip_last: %s", inet_ntoa(temp)); + + // process the source port range that ZMap is allowed to use + num_src_ports = zconf.source_port_last - zconf.source_port_first + 1; + log_debug("send", "will send from %u address%s on %hu source ports", + zconf.number_source_ips, + ((zconf.number_source_ips == 1) ? "" : "es"), num_src_ports); + // global initialization for send module + assert(zconf.probe_module); + if (zconf.probe_module->global_initialize) { + if (zconf.probe_module->global_initialize(&zconf)) { + log_fatal( + "send", + "global initialization for probe module failed."); + } + } + + // only allow bandwidth or rate + if (zconf.bandwidth > 0 && zconf.rate > 0) { + log_fatal( + "send", + "must specify rate or bandwidth, or neither, not both."); + } + + // Convert specified bandwidth to packet rate. This is an estimate using the + // max packet size a probe module will generate. + if (zconf.bandwidth > 0) { + size_t pkt_len = zconf.probe_module->max_packet_length; + pkt_len *= 8; + // 7 byte MAC preamble, 1 byte Start frame, 4 byte CRC, 12 byte + // inter-frame gap + pkt_len += 8 * 24; + // adjust calculated length if less than the minimum size of an + // ethernet frame + if (pkt_len < 84 * 8) { + pkt_len = 84 * 8; + } + // rate is a uint32_t so, don't overflow + if (zconf.bandwidth / pkt_len > 0xFFFFFFFFu) { + zconf.rate = 0; + } else { + zconf.rate = zconf.bandwidth / pkt_len; + if (zconf.rate == 0) { + log_warn( + "send", + "bandwidth %lu bit/s is slower than 1 pkt/s, " + "setting rate to 1 pkt/s", + zconf.bandwidth); + zconf.rate = 1; + } + } + log_debug( + "send", + "using bandwidth %lu bits/s for %zu byte probe, rate set to %d pkt/s", + zconf.bandwidth, pkt_len / 8, zconf.rate); + } + // convert default placeholder to default value + if (zconf.rate == -1) { + // default 10K pps + zconf.rate = 10000; + } + // log rate, if explicitly specified + if (zconf.rate < 0) { + log_fatal("send", "rate impossibly slow"); + } + if (zconf.rate > 0 && zconf.bandwidth <= 0) { + log_debug("send", "rate set to %d pkt/s", zconf.rate); + } + // Get the source hardware address, and give it to the probe + // module + if (!zconf.hw_mac_set) { + if (get_iface_hw_addr(zconf.iface, zconf.hw_mac)) { + log_fatal( + "send", + "ZMap could not retrieve the hardware (MAC) address for " + "the interface \"%s\". You likely do not privileges to open a raw packet socket. " + "Are you running as root or with the CAP_NET_RAW capability? If you are, you " + "may need to manually set the source MAC address with the \"--source-mac\" flag.", + zconf.iface); + return NULL; + } + log_debug( + "send", + "no source MAC provided. " + "automatically detected %02x:%02x:%02x:%02x:%02x:%02x as hw " + "interface for %s", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5], + zconf.iface); + } + log_debug("send", "source MAC address %02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + + if (zconf.dryrun) { + log_info("send", "dryrun mode -- won't actually send packets"); + } + // initialize random validation key + validate_init(); + // setup signal handlers for changing scan speed + signal(SIGUSR1, sig_handler_increase_speed); + signal(SIGUSR2, sig_handler_decrease_speed); + zsend.start = now(); + return it; +} + +static inline ipaddr_n_t get_src_ip(ipaddr_n_t dst, int local_offset) +{ + if (zconf.number_source_ips == 1) { + return zconf.source_ip_addresses[0]; + } + return zconf.source_ip_addresses[(ntohl(dst) + local_offset) % + zconf.number_source_ips]; +} + +// one sender thread +int send_run(sock_t st, shard_t *s) +{ + log_debug("send", "send thread started"); + pthread_mutex_lock(&send_mutex); + // Allocate a buffer to hold the outgoing packet + char buf[MAX_PACKET_SIZE]; + memset(buf, 0, MAX_PACKET_SIZE); + + // OS specific per-thread init + if (send_run_init(st)) { + pthread_mutex_unlock(&send_mutex); + return -1; + } + // MAC address length in characters + char mac_buf[(ETHER_ADDR_LEN * 2) + (ETHER_ADDR_LEN - 1) + 1]; + char *p = mac_buf; + for (int i = 0; i < ETHER_ADDR_LEN; i++) { + if (i == ETHER_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.hw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.hw_mac[i]); + p += 3; + } + } + log_debug("send", "source MAC address %s", mac_buf); + void *probe_data; + if (zconf.probe_module->thread_initialize) { + zconf.probe_module->thread_initialize( + buf, zconf.hw_mac, zconf.gw_mac, zconf.target_port, + &probe_data); + } + pthread_mutex_unlock(&send_mutex); + + // adaptive timing to hit target rate + uint64_t count = 0; + uint64_t last_count = count; + double last_time = now(); + uint32_t delay = 0; + int interval = 0; + volatile int vi; + struct timespec ts, rem; + double send_rate = + (double)zconf.rate / + ((double)zconf.senders * zconf.batch * zconf.packet_streams); + const double slow_rate = 50; // packets per seconds per thread + // at which it uses the slow methods + long nsec_per_sec = 1000 * 1000 * 1000; + long long sleep_time = nsec_per_sec; + if (zconf.rate > 0) { + delay = 10000; + if (send_rate < slow_rate) { + // set the initial time difference + sleep_time = nsec_per_sec / send_rate; + last_time = now() - (1.0 / send_rate); + } else { + // estimate initial rate + for (vi = delay; vi--;) + ; + delay *= 1 / (now() - last_time) / + ((double)zconf.rate / + ((double)zconf.senders * zconf.batch)); + interval = ((double)zconf.rate / + ((double)zconf.senders * zconf.batch)) / + 20; + last_time = now(); + } + } + // Get the initial IP to scan. + uint32_t current_ip = shard_get_cur_ip(s); + + // If provided a list of IPs to scan, then the first generated address + // might not be on that list. Iterate until the current IP is one the + // list, then start the true scanning process. + if (zconf.list_of_ips_filename) { + while (!pbm_check(zsend.list_of_ips_pbm, current_ip)) { + current_ip = shard_get_next_ip(s); + if (current_ip == ZMAP_SHARD_DONE) { + log_debug( + "send", + "never made it to send loop in send thread %i", + s->thread_id); + goto cleanup; + } + } + } + int attempts = zconf.num_retries + 1; + uint32_t idx = 0; + while (1) { + // Adaptive timing delay + if (count && delay > 0) { + if (send_rate < slow_rate) { + double t = now(); + double last_rate = (1.0 / (t - last_time)); + + sleep_time *= ((last_rate / send_rate) + 1) / 2; + ts.tv_sec = sleep_time / nsec_per_sec; + ts.tv_nsec = sleep_time % nsec_per_sec; + log_debug("sleep", + "sleep for %d sec, %ld nanoseconds", + ts.tv_sec, ts.tv_nsec); + while (nanosleep(&ts, &rem) == -1) { + } + last_time = t; + } else { + for (vi = delay; vi--;) + ; + if (!interval || (count % interval == 0)) { + double t = now(); + assert(count > last_count); + assert(t > last_time); + double multiplier = + (double)(count - last_count) / + (t - last_time) / + (zconf.rate / zconf.senders); + uint32_t old_delay = delay; + delay *= multiplier; + if (delay == old_delay) { + if (multiplier > 1.0) { + delay *= 2; + } else if (multiplier < 1.0) { + delay *= 0.5; + } + } + last_count = count; + last_time = t; + } + } + } + + // Check if the program has otherwise completed and break out of the send loop. + if (zrecv.complete) { + goto cleanup; + } + if (zconf.max_runtime && + zconf.max_runtime <= now() - zsend.start) { + goto cleanup; + } + + // Actually send a packet. + for (int b = 0; b < zconf.batch; b++) { + // Check if we've finished this shard or thread before sending each + // packet, regardless of batch size. + if (s->state.max_hosts && + s->state.hosts_scanned >= s->state.max_hosts) { + log_debug( + "send", + "send thread %hhu finished (max targets of %u reached)", + s->thread_id, s->state.max_hosts); + goto cleanup; + } + if (s->state.max_packets && + s->state.packets_sent >= s->state.max_packets) { + log_debug( + "send", + "send thread %hhu finished (max packets of %u reached)", + s->thread_id, s->state.max_packets); + goto cleanup; + } + if (current_ip == ZMAP_SHARD_DONE) { + log_debug( + "send", + "send thread %hhu finished, shard depleted", + s->thread_id); + goto cleanup; + } + for (int i = 0; i < zconf.packet_streams; i++) { + count++; + uint32_t src_ip = get_src_ip(current_ip, i); + uint32_t validation[VALIDATE_BYTES / + sizeof(uint32_t)]; + validate_gen(src_ip, current_ip, + (uint8_t *)validation); + uint8_t ttl = zconf.probe_ttl; + size_t length = 0; + zconf.probe_module->make_packet( + buf, &length, src_ip, current_ip, ttl, + validation, i, probe_data); + if (length > MAX_PACKET_SIZE) { + log_fatal( + "send", + "send thread %hhu set length (%zu) larger than MAX (%zu)", + s->thread_id, length, + MAX_PACKET_SIZE); + } + if (zconf.dryrun) { + lock_file(stdout); + zconf.probe_module->print_packet(stdout, + buf); + unlock_file(stdout); + } else { + void *contents = + buf + + zconf.send_ip_pkts * + sizeof(struct ether_header); + length -= (zconf.send_ip_pkts * + sizeof(struct ether_header)); + int any_sends_successful = 0; + for (int i = 0; i < attempts; ++i) { + int rc = send_packet( + st, contents, length, idx); + if (rc < 0) { + struct in_addr addr; + addr.s_addr = + current_ip; + char addr_str_buf + [INET_ADDRSTRLEN]; + const char *addr_str = + inet_ntop( + AF_INET, &addr, + addr_str_buf, + INET_ADDRSTRLEN); + if (addr_str != NULL) { + log_debug( + "send", + "send_packet failed for %s. %s", + addr_str, + strerror( + errno)); + } + } else { + any_sends_successful = + 1; + break; + } + } + if (!any_sends_successful) { + s->state.packets_failed++; + } + idx++; + idx &= 0xFF; + } + s->state.packets_sent++; + } + // Track the number of hosts we actually scanned. + s->state.hosts_scanned++; + + // Get the next IP to scan + current_ip = shard_get_next_ip(s); + if (zconf.list_of_ips_filename && + current_ip != ZMAP_SHARD_DONE) { + // If we have a list of IPs bitmap, ensure the next IP + // to scan is on the list. + while (!pbm_check(zsend.list_of_ips_pbm, + current_ip)) { + current_ip = shard_get_next_ip(s); + if (current_ip == ZMAP_SHARD_DONE) { + log_debug( + "send", + "send thread %hhu shard finished in get_next_ip_loop depleted", + s->thread_id); + goto cleanup; + } + } + } + } + } +cleanup: + s->cb(s->thread_id, s->arg); + if (zconf.dryrun) { + lock_file(stdout); + fflush(stdout); + unlock_file(stdout); + } + log_debug("send", "thread %hu cleanly finished", s->thread_id); + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/send.h b/rdns_scan/zmap4rdns/src/send.h new file mode 100644 index 0000000..eff72f3 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/send.h @@ -0,0 +1,18 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef SEND_H +#define SEND_H + +#include "iterator.h" +#include "socket.h" + +iterator_t *send_init(void); +int send_run(sock_t, shard_t *); + +#endif // SEND_H diff --git a/rdns_scan/zmap4rdns/src/shard.c b/rdns_scan/zmap4rdns/src/shard.c new file mode 100644 index 0000000..526cc3c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/shard.c @@ -0,0 +1,161 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdint.h> +#include <assert.h> + +#include <gmp.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "shard.h" +#include "state.h" + +static uint32_t shard_roll_to_valid(shard_t *s) +{ + if (s->current - 1 < zsend.max_index) { + return s->current; + } + return shard_get_next_ip(s); +} + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint32_t max_total_targets, const cycle_t *cycle, + shard_complete_cb cb, void *arg) +{ + // Start out by figuring out how many shards we have. A single shard of + // ZMap (set with --shards=N, --shard=n) may have several subshards, if + // ZMap is being ran multithreaded (set with --sender-threads=T). + // + // Total number of subshards is S = N*T. Subshard ID's range from [0, + // N*T). + assert(num_shards > 0); + assert(num_threads > 0); + assert(shard_idx < num_shards); + assert(thread_idx < num_threads); + uint32_t num_subshards = (uint32_t)num_shards * (uint32_t)num_threads; + uint64_t num_elts = cycle->order; + assert(num_subshards < num_elts); + assert(!max_total_targets || (num_subshards <= max_total_targets)); + + // This instance of ZMap will run T subshards, with one subshard per + // thread. This composes a single shard, as specified by the command + // line flag --shard=n. E.g. to run shard with index n, we must run + // subshards with indices the range [n*T, (n+1)*T]. + // + // We can calculate our subshard index i = n*T + t. + uint32_t sub_idx = shard_idx * num_threads + thread_idx; + + // Given i, we want to calculate the start of subshard i. Subshards + // define ranges over exponents of g. They range from [0, Q-1), where Q + // is the number of elements in (the order of) the group generated by + // g. + // + // Let e_b = floor(Q / S) * i + uint64_t exponent_begin = (num_elts / num_subshards) * sub_idx; + + // The stopping exponent is the first element of the next shard. + // + // e_e = floor(Q / S) * ((i + 1) % S) + uint64_t exponent_end = + (num_elts / num_subshards) * ((sub_idx + 1) % num_subshards); + + // We actually offset the begin and end of each cycle. Given an offset + // k, shift each exponent by k modulo Q. + exponent_begin = (exponent_begin + cycle->offset) % num_elts; + exponent_end = (exponent_end + cycle->offset) % num_elts; + + // Multiprecision variants of everything above + mpz_t generator_m, exponent_begin_m, exponent_end_m, prime_m; + mpz_init_set_ui(generator_m, cycle->generator); + mpz_init_set_ui(exponent_begin_m, exponent_begin); + mpz_init_set_ui(exponent_end_m, exponent_end); + mpz_init_set_ui(prime_m, cycle->group->prime); + + // Calculate the first and last points of the shard as powers of g + // modulo p. + mpz_t start_m, stop_m; + mpz_init(start_m); + mpz_init(stop_m); + mpz_powm(start_m, generator_m, exponent_begin_m, prime_m); + mpz_powm(stop_m, generator_m, exponent_end_m, prime_m); + + // Pull the result out as a uint64_t + shard->params.first = (uint64_t)mpz_get_ui(start_m); + shard->params.last = (uint64_t)mpz_get_ui(stop_m); + shard->params.factor = cycle->generator; + shard->params.modulus = cycle->group->prime; + + // Set the shard at the beginning. + shard->current = shard->params.first; + + // Set the (thread) id + shard->thread_id = thread_idx; + + // Set max_targets if applicable + if (max_total_targets > 0) { + uint32_t max_targets_this_shard = + max_total_targets / num_subshards; + if (sub_idx < (max_total_targets % num_subshards)) { + ++max_targets_this_shard; + } + shard->state.max_hosts = max_targets_this_shard; + } + + // Set the callbacks + shard->cb = cb; + shard->arg = arg; + + // If the beginning of a shard isn't pointing to a valid index in the + // blocklist, find the first element that is. + shard_roll_to_valid(shard); + + // Clear everything + mpz_clear(generator_m); + mpz_clear(exponent_begin_m); + mpz_clear(exponent_end_m); + mpz_clear(prime_m); + mpz_clear(start_m); + mpz_clear(stop_m); +} + +uint32_t shard_get_cur_ip(shard_t *shard) +{ + return (uint32_t)blocklist_lookup_index(shard->current - 1); +} + +static inline uint32_t shard_get_next_elem(shard_t *shard) +{ + do { + shard->current *= shard->params.factor; + shard->current %= shard->params.modulus; + } while (shard->current >= (1LL << 32)); + return (uint32_t)shard->current; +} + +uint32_t shard_get_next_ip(shard_t *shard) +{ + if (shard->current == ZMAP_SHARD_DONE) { + return ZMAP_SHARD_DONE; + } + while (1) { + uint32_t candidate = shard_get_next_elem(shard); + if (candidate == shard->params.last) { + shard->current = ZMAP_SHARD_DONE; + shard->iterations++; + return ZMAP_SHARD_DONE; + } + if (candidate - 1 < zsend.max_index) { + shard->state.hosts_allowlisted++; + shard->iterations++; + return blocklist_lookup_index(candidate - 1); + } + shard->state.hosts_blocklisted++; + } +} diff --git a/rdns_scan/zmap4rdns/src/shard.h b/rdns_scan/zmap4rdns/src/shard.h new file mode 100644 index 0000000..19f0381 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/shard.h @@ -0,0 +1,52 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SHARD_H +#define ZMAP_SHARD_H + +#include <stdint.h> + +#include "cyclic.h" + +#define ZMAP_SHARD_DONE 0 + +typedef void (*shard_complete_cb)(uint8_t id, void *arg); + +typedef struct shard { + struct shard_state { + uint64_t packets_sent; + uint32_t hosts_scanned; + uint32_t max_hosts; + uint32_t max_packets; + uint32_t hosts_blocklisted; + uint32_t hosts_allowlisted; + uint32_t packets_failed; + uint32_t first_scanned; + } state; + struct shard_params { + uint64_t first; + uint64_t last; + uint64_t factor; + uint64_t modulus; + } params; + uint64_t current; + uint64_t iterations; + uint8_t thread_id; + shard_complete_cb cb; + void *arg; +} shard_t; + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint32_t max_total_targets, const cycle_t *cycle, + shard_complete_cb cb, void *arg); + +uint32_t shard_get_cur_ip(shard_t *shard); +uint32_t shard_get_next_ip(shard_t *shard); + +#endif /* ZMAP_SHARD_H */ diff --git a/rdns_scan/zmap4rdns/src/socket-bsd.c b/rdns_scan/zmap4rdns/src/socket-bsd.c new file mode 100644 index 0000000..e55cd72 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/socket-bsd.c @@ -0,0 +1,66 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include <errno.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include <sys/types.h> +#include <sys/time.h> +#include <sys/ioctl.h> +#include <fcntl.h> +#include <net/bpf.h> + +#include "state.h" + +sock_t get_socket(UNUSED uint32_t id) +{ + char file[32]; + int bpf; + // Assume failure + sock_t ret; + ret.sock = -1; + + if(zconf.send_ip_pkts && !zconf.dryrun) { + log_fatal("socket", "iplayer not supported on bsd"); + } + + // Try to find a valid bpf + for (int i = 0; i < 128; i++) { + snprintf(file, sizeof(file), "/dev/bpf%d", i); + bpf = open(file, O_WRONLY); + if (bpf != -1 || errno != EBUSY) + break; + } + + // Make sure it worked + if (bpf < 0) { + return ret; + } + + // Set up an ifreq to bind to + struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); + strlcpy(ifr.ifr_name, zconf.iface, sizeof(ifr.ifr_name)); + + // Bind the bpf to the interface + if (ioctl(bpf, BIOCSETIF, (char *)&ifr) < 0) { + return ret; + } + + // Enable writing the address in + int write_addr_enable = 1; + if (ioctl(bpf, BIOCSHDRCMPLT, &write_addr_enable) < 0) { + return ret; + } + ret.sock = bpf; + return ret; +} diff --git a/rdns_scan/zmap4rdns/src/socket-linux.c b/rdns_scan/zmap4rdns/src/socket-linux.c new file mode 100644 index 0000000..b1710c7 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/socket-linux.c @@ -0,0 +1,36 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include <string.h> +#include <errno.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" + +#include "state.h" + +sock_t get_socket(UNUSED uint32_t id) +{ + int sock; + if (zconf.send_ip_pkts) { + sock = socket(AF_PACKET, SOCK_DGRAM, htons(ETH_P_ALL)); + } else { + sock = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); + } + if (sock <= 0) { + log_fatal("send", + "couldn't create socket. " + "Are you root? Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + return s; +} diff --git a/rdns_scan/zmap4rdns/src/socket-pfring.c b/rdns_scan/zmap4rdns/src/socket-pfring.c new file mode 100644 index 0000000..a8f3c7e --- /dev/null +++ b/rdns_scan/zmap4rdns/src/socket-pfring.c @@ -0,0 +1,22 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include "../lib/includes.h" +#include "state.h" + +#include <pfring_zc.h> + +sock_t get_socket(uint32_t id) +{ + sock_t sock; + sock.pf.queue = zconf.pf.queues[id]; + sock.pf.buffers = zconf.pf.buffers + 256 * id; + return sock; +} diff --git a/rdns_scan/zmap4rdns/src/socket.c b/rdns_scan/zmap4rdns/src/socket.c new file mode 100644 index 0000000..f6a877e --- /dev/null +++ b/rdns_scan/zmap4rdns/src/socket.c @@ -0,0 +1,31 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include <string.h> +#include <errno.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" + +sock_t get_dryrun_socket(void) +{ + // we need a socket in order to gather details about the system + // such as source MAC address and IP address. However, because + // we don't want to require root access in order to run dryrun, + // we just create a TCP socket. + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock <= 0) { + log_fatal("send", "couldn't create socket. Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + return s; +} diff --git a/rdns_scan/zmap4rdns/src/socket.h b/rdns_scan/zmap4rdns/src/socket.h new file mode 100644 index 0000000..8b9fe84 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/socket.h @@ -0,0 +1,40 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef ZMAP_SOCKET_H +#define ZMAP_SOCKET_H + +#include <stdint.h> + +#include "../lib/includes.h" + +#ifdef PFRING + +#include <pfring_zc.h> + +typedef union { + int sock; + struct { + pfring_zc_queue *queue; + pfring_zc_pkt_buff **buffers; + int idx; + } pf; +} sock_t; + +#else + +typedef struct { + int sock; +} sock_t; + +#endif /* PFRING */ + +sock_t get_dryrun_socket(void); +sock_t get_socket(uint32_t id); + +#endif /* ZMAP_SOCKET_H */ diff --git a/rdns_scan/zmap4rdns/src/state.c b/rdns_scan/zmap4rdns/src/state.c new file mode 100644 index 0000000..a279343 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/state.c @@ -0,0 +1,104 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "state.h" +#include "../lib/logger.h" + +// global configuration and defaults +struct state_conf zconf = {.log_level = LOG_INFO, + .source_port_first = 32768, // (these are the default + .source_port_last = 61000, // ephemeral range on Linux) + .output_filename = NULL, + .blocklist_filename = NULL, + .allowlist_filename = NULL, + .list_of_ips_filename = NULL, + .list_of_ips_count = 0, + .target_port = 0, + .max_targets = 0xFFFFFFFF, + .max_runtime = 0, + .max_results = 0, + .iface = NULL, + .rate = -1, + .bandwidth = 0, + .cooldown_secs = 0, + .senders = 1, + .batch = 1, + .packet_streams = 1, + .seed_provided = 0, + .seed = 0, + .output_module = NULL, + .output_args = NULL, + .probe_module = NULL, + .probe_args = NULL, + .probe_ttl = MAXTTL, + .gw_mac = {0}, + .gw_ip = 0, + .hw_mac = {0}, + .gw_mac_set = 0, + .hw_mac_set = 0, + .number_source_ips = 0, + .send_ip_pkts = 0, + .raw_output_fields = NULL, + .output_fields = NULL, + .output_filter_str = NULL, + .output_fields_len = 0, + .log_file = NULL, + .log_directory = NULL, + .status_updates_file = NULL, + .dryrun = 0, + .quiet = 0, + .syslog = 1, + .max_sendto_failures = -1, + .min_hitrate = 0.0, + .metadata_file = NULL, + .metadata_filename = NULL, + .notes = NULL, + .custom_metadata_str = NULL, + .recv_ready = 0, + .data_link_size = 0, + .default_mode = 0, + .no_header_row = 0, +}; + +void init_empty_global_configuration(struct state_conf *c) { + memset(c->source_ip_addresses, 0, sizeof(c->source_ip_addresses)); +} + +// global sender stats and defaults +struct state_send zsend = { + .start = 0.0, + .finish = 0.0, + .packets_sent = 0, + .hosts_scanned = 0, + .blocklisted = 0, + .allowlisted = 0, + .warmup = 1, + .complete = 0, + .sendto_failures = 0, + .max_targets = 0, + .list_of_ips_pbm = NULL, +}; + +// global receiver stats and defaults +struct state_recv zrecv = { + .success_unique = 0, + .success_total = 0, + .app_success_unique = 0, + .app_success_total = 0, + .validation_passed = 0, + .validation_failed = 0, + .cooldown_unique = 0, + .cooldown_total = 0, + .failure_total = 0, + .filter_success = 0, + .ip_fragments = 0, + .complete = 0, + .pcap_recv = 0, + .pcap_drop = 0, + .pcap_ifdrop = 0, +}; diff --git a/rdns_scan/zmap4rdns/src/state.h b/rdns_scan/zmap4rdns/src/state.h new file mode 100644 index 0000000..68e4c33 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/state.h @@ -0,0 +1,202 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef STATE_H +#define STATE_H + +#include <stdio.h> +#include <stdint.h> + +#include "../lib/includes.h" + +#ifdef PFRING +#include <pfring_zc.h> +#endif + +#include "aesrand.h" +#include "fieldset.h" +#include "filter.h" +#include "types.h" + +#define MAX_PACKET_SIZE 4096 +#define MAC_ADDR_LEN_BYTES 6 + +struct probe_module; +struct output_module; + +struct fieldset_conf { + fielddefset_t defs; + fielddefset_t outdefs; + translation_t translation; + int success_index; + int app_success_index; + int classification_index; +}; + +// global configuration +struct state_conf { + int log_level; + port_h_t target_port; + port_h_t source_port_first; + port_h_t source_port_last; + // maximum number of packets that the scanner will send before + // terminating + uint32_t max_targets; + // maximum number of seconds that scanner will run before terminating + uint32_t max_runtime; + // maximum number of results before terminating + uint32_t max_results; + // name of network interface that + // will be utilized for sending/receiving + char *iface; + // rate in packets per second + // that the sender will maintain + int rate; + // rate in bits per second + uint64_t bandwidth; + // how many seconds after the termination of the sender will the + // receiver continue to process responses + int cooldown_secs; + // number of sending threads + uint8_t senders; + uint8_t batch; + uint32_t pin_cores_len; + uint32_t *pin_cores; + // should use CLI provided randomization seed instead of generating + // a random seed. + int seed_provided; + uint64_t seed; + aesrand_t *aes; + // generator of the cyclic multiplicative group that is utilized for + // address generation + uint32_t generator; + // sharding options + uint16_t shard_num; + uint16_t total_shards; + int packet_streams; + struct probe_module *probe_module; + char *output_module_name; + struct output_module *output_module; + char *probe_args; + uint8_t probe_ttl; + char *output_args; + macaddr_t gw_mac[MAC_ADDR_LEN_BYTES]; + macaddr_t hw_mac[MAC_ADDR_LEN_BYTES]; + uint32_t gw_ip; + int gw_mac_set; + int hw_mac_set; + in_addr_t source_ip_addresses[256]; + uint32_t number_source_ips; + int send_ip_pkts; + char *output_filename; + char *blocklist_filename; + char *allowlist_filename; + char *list_of_ips_filename; + uint32_t list_of_ips_count; + char *metadata_filename; + FILE *metadata_file; + char *notes; + char *custom_metadata_str; + char **destination_cidrs; + int destination_cidrs_len; + const char *raw_output_fields; + const char **output_fields; + struct output_filter filter; + char *output_filter_str; + struct fieldset_conf fsconf; + int output_fields_len; + char *log_file; + char *log_directory; + char *status_updates_file; + int dryrun; + int quiet; + int ignore_invalid_hosts; + int syslog; + int recv_ready; + int num_retries; + uint64_t total_allowed; + uint64_t total_disallowed; + int max_sendto_failures; + float min_hitrate; + int data_link_size; + int default_mode; + int no_header_row; +#ifdef PFRING + struct { + pfring_zc_cluster *cluster; + pfring_zc_queue *send; + pfring_zc_queue *recv; + pfring_zc_queue **queues; + pfring_zc_pkt_buff **buffers; + pfring_zc_buffer_pool *prefetches; + } pf; +#endif +}; +extern struct state_conf zconf; + +void init_empty_global_configuration(struct state_conf *c); + +// global sender stats +struct state_send { + double start; + double finish; + uint64_t packets_sent; + uint64_t hosts_scanned; + uint64_t blocklisted; + uint64_t allowlisted; + int warmup; + int complete; + uint32_t first_scanned; + uint32_t max_targets; + uint32_t sendto_failures; + uint32_t max_index; + uint8_t **list_of_ips_pbm; +}; +extern struct state_send zsend; + +// global receiver stats +struct state_recv { + // valid responses classified as "success" + uint32_t success_total; + // unique IPs that sent valid responses classified as "success" + uint32_t success_unique; + // valid responses classified as "success" + uint32_t app_success_total; + // unique IPs that sent valid responses classified as "success" + uint32_t app_success_unique; + // valid responses classified as "success" received during cooldown + uint32_t cooldown_total; + // unique IPs that first sent valid "success"es during cooldown + uint32_t cooldown_unique; + // valid responses NOT classified as "success" + uint32_t failure_total; + // valid responses that passed the filter + uint64_t filter_success; + // how many packets did we receive that were marked as being the first + // fragment in a stream + uint32_t ip_fragments; + // metrics about _only_ validate_packet + uint32_t validation_passed; + uint32_t validation_failed; + + int complete; // has the scanner finished sending? + double start; // timestamp of when recv started + double finish; // timestamp of when recv terminated + + // number of packets captured by pcap filter + uint32_t pcap_recv; + // number of packets dropped because there was no room in + // the operating system's buffer when they arrived, because + // packets weren't being read fast enough + uint32_t pcap_drop; + // number of packets dropped by the network interface or its driver. + uint32_t pcap_ifdrop; +}; +extern struct state_recv zrecv; + +#endif // _STATE_H diff --git a/rdns_scan/zmap4rdns/src/summary.c b/rdns_scan/zmap4rdns/src/summary.c new file mode 100644 index 0000000..d152ac6 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/summary.c @@ -0,0 +1,346 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "summary.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <unistd.h> + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/blocklist.h" + +#include "state.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/output_modules.h" + +#define STRTIME_LEN 1024 + +#include <json.h> + +void json_metadata(FILE *file) +{ + char send_start_time[STRTIME_LEN + 1]; + assert(dstrftime(send_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zsend.start)); + char send_end_time[STRTIME_LEN + 1]; + assert(dstrftime(send_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zsend.finish)); + char recv_start_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zrecv.start)); + char recv_end_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + zrecv.finish)); + double hitrate = + ((double)100 * zrecv.success_unique) / ((double)zsend.hosts_scanned); + + json_object *obj = json_object_new_object(); + + // scanner host name + char hostname[1024]; + if (gethostname(hostname, 1023) < 0) { + log_error("json_metadata", "unable to retrieve local hostname"); + } else { + hostname[1023] = '\0'; + json_object_object_add(obj, "local_hostname", + json_object_new_string(hostname)); + struct hostent *h = gethostbyname(hostname); + if (h) { + json_object_object_add( + obj, "full_hostname", + json_object_new_string(h->h_name)); + } else { + log_error("json_metadata", + "unable to retrieve complete hostname"); + } + } + + json_object_object_add(obj, "target_port", + json_object_new_int(zconf.target_port)); + json_object_object_add(obj, "source_port_first", + json_object_new_int(zconf.source_port_first)); + json_object_object_add(obj, "source_port_last", + json_object_new_int(zconf.source_port_last)); + json_object_object_add(obj, "max_targets", + json_object_new_int(zconf.max_targets)); + json_object_object_add(obj, "max_runtime", + json_object_new_int(zconf.max_runtime)); + json_object_object_add(obj, "max_results", + json_object_new_int(zconf.max_results)); + json_object_object_add(obj, "output_results", + json_object_new_int(zrecv.filter_success)); + if (zconf.iface) { + json_object_object_add(obj, "iface", + json_object_new_string(zconf.iface)); + } + json_object_object_add(obj, "rate", json_object_new_int(zconf.rate)); + json_object_object_add(obj, "bandwidth", + json_object_new_int(zconf.bandwidth)); + json_object_object_add(obj, "cooldown_secs", + json_object_new_int(zconf.cooldown_secs)); + json_object_object_add(obj, "senders", + json_object_new_int(zconf.senders)); + json_object_object_add(obj, "seed", json_object_new_int64(zconf.seed)); + json_object_object_add(obj, "seed_provided", + json_object_new_int64(zconf.seed_provided)); + json_object_object_add(obj, "generator", + json_object_new_int64(zconf.generator)); + json_object_object_add(obj, "hitrate", json_object_new_double(hitrate)); + json_object_object_add(obj, "shard_num", + json_object_new_int(zconf.shard_num)); + json_object_object_add(obj, "total_shards", + json_object_new_int(zconf.total_shards)); + + json_object_object_add(obj, "min_hitrate", + json_object_new_double(zconf.min_hitrate)); + json_object_object_add(obj, "max_sendto_failures", + json_object_new_int(zconf.max_sendto_failures)); + + json_object_object_add(obj, "syslog", + json_object_new_int(zconf.syslog)); + json_object_object_add(obj, "default_mode", + json_object_new_int(zconf.default_mode)); + json_object_object_add(obj, "pcap_recv", + json_object_new_int(zrecv.pcap_recv)); + json_object_object_add(obj, "pcap_drop", + json_object_new_int(zrecv.pcap_drop)); + json_object_object_add(obj, "pcap_ifdrop", + json_object_new_int(zrecv.pcap_ifdrop)); + + json_object_object_add(obj, "ip_fragments", + json_object_new_int(zrecv.ip_fragments)); + json_object_object_add(obj, "blocklist_total_allowed", + json_object_new_int64(zconf.total_allowed)); + json_object_object_add(obj, "blocklist_total_not_allowed", + json_object_new_int64(zconf.total_disallowed)); + json_object_object_add(obj, "validation_passed", + json_object_new_int(zrecv.validation_passed)); + json_object_object_add(obj, "validation_failed", + json_object_new_int(zrecv.validation_failed)); + + // json_object_object_add(obj, "blocklisted", + // json_object_new_int64(zsend.blocklisted)); + // json_object_object_add(obj, "allowlisted", + // json_object_new_int64(zsend.allowlisted)); + json_object_object_add(obj, "first_scanned", + json_object_new_int64(zsend.first_scanned)); + json_object_object_add(obj, "send_to_failures", + json_object_new_int64(zsend.sendto_failures)); + json_object_object_add(obj, "packets_sent", + json_object_new_int64(zsend.packets_sent)); + json_object_object_add(obj, "hosts_scanned", + json_object_new_int64(zsend.hosts_scanned)); + json_object_object_add(obj, "success_total", + json_object_new_int64(zrecv.success_total)); + json_object_object_add(obj, "success_unique", + json_object_new_int64(zrecv.success_unique)); + if (zconf.fsconf.app_success_index >= 0) { + json_object_object_add( + obj, "app_success_total", + json_object_new_int64(zrecv.app_success_total)); + json_object_object_add( + obj, "app_success_unique", + json_object_new_int64(zrecv.app_success_unique)); + } + json_object_object_add(obj, "success_cooldown_total", + json_object_new_int64(zrecv.cooldown_total)); + json_object_object_add(obj, "success_cooldown_unique", + json_object_new_int64(zrecv.cooldown_unique)); + json_object_object_add(obj, "failure_total", + json_object_new_int64(zrecv.failure_total)); + + json_object_object_add(obj, "packet_streams", + json_object_new_int(zconf.packet_streams)); + json_object_object_add( + obj, "probe_module", + json_object_new_string( + ((probe_module_t *)zconf.probe_module)->name)); + json_object_object_add( + obj, "output_module", + json_object_new_string( + ((output_module_t *)zconf.output_module)->name)); + + json_object_object_add(obj, "send_start_time", + json_object_new_string(send_start_time)); + json_object_object_add(obj, "send_end_time", + json_object_new_string(send_end_time)); + json_object_object_add(obj, "recv_start_time", + json_object_new_string(recv_start_time)); + json_object_object_add(obj, "recv_end_time", + json_object_new_string(recv_end_time)); + + if (zconf.output_filter_str) { + json_object_object_add( + obj, "output_filter", + json_object_new_string(zconf.output_filter_str)); + } + if (zconf.log_file) { + json_object_object_add(obj, "log_file", + json_object_new_string(zconf.log_file)); + } + if (zconf.log_directory) { + json_object_object_add( + obj, "log_directory", + json_object_new_string(zconf.log_directory)); + } + + if (zconf.destination_cidrs_len) { + json_object *cli_dest_cidrs = json_object_new_array(); + for (int i = 0; i < zconf.destination_cidrs_len; i++) { + json_object_array_add( + cli_dest_cidrs, + json_object_new_string(zconf.destination_cidrs[i])); + } + json_object_object_add(obj, "cli_cidr_destinations", + cli_dest_cidrs); + } + if (zconf.probe_args) { + json_object_object_add( + obj, "probe_args", + json_object_new_string(zconf.probe_args)); + } + if (zconf.probe_ttl) { + json_object_object_add( + obj, "probe_ttl", + json_object_new_int(zconf.probe_ttl)); + } + if (zconf.output_args) { + json_object_object_add( + obj, "output_args", + json_object_new_string(zconf.output_args)); + } + { + char mac_buf[(MAC_ADDR_LEN * 2) + (MAC_ADDR_LEN - 1) + 1]; + memset(mac_buf, 0, sizeof(mac_buf)); + char *p = mac_buf; + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i == MAC_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.gw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.gw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "gateway_mac", + json_object_new_string(mac_buf)); + } + if (zconf.gw_ip) { + struct in_addr addr; + addr.s_addr = zconf.gw_ip; + json_object_object_add(obj, "gateway_ip", + json_object_new_string(inet_ntoa(addr))); + } + { + char mac_buf[(ETHER_ADDR_LEN * 2) + (ETHER_ADDR_LEN - 1) + 1]; + char *p = mac_buf; + for (int i = 0; i < ETHER_ADDR_LEN; i++) { + if (i == ETHER_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", zconf.hw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", zconf.hw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "source_mac", + json_object_new_string(mac_buf)); + } + json_object *source_ips = json_object_new_array(); + for (uint i = 0; i < zconf.number_source_ips; i++) { + struct in_addr temp; + temp.s_addr = zconf.source_ip_addresses[i]; + json_object_array_add(source_ips, json_object_new_string( + strdup(inet_ntoa(temp)))); + } + json_object_object_add(obj, "source_ips", source_ips); + if (zconf.output_filename) { + json_object_object_add( + obj, "output_filename", + json_object_new_string(zconf.output_filename)); + } + if (zconf.blocklist_filename) { + json_object_object_add( + obj, "blocklist_filename", + json_object_new_string(zconf.blocklist_filename)); + } + if (zconf.allowlist_filename) { + json_object_object_add( + obj, "allowlist_filename", + json_object_new_string(zconf.allowlist_filename)); + } + if (zconf.list_of_ips_filename) { + json_object_object_add( + obj, "list_of_ips_filename", + json_object_new_string(zconf.list_of_ips_filename)); + json_object_object_add( + obj, "list_of_ips_count", + json_object_new_int(zconf.list_of_ips_count)); + } + json_object_object_add(obj, "dryrun", + json_object_new_int(zconf.dryrun)); + json_object_object_add(obj, "quiet", json_object_new_int(zconf.quiet)); + json_object_object_add(obj, "log_level", + json_object_new_int(zconf.log_level)); + + // parse out JSON metadata that was supplied on the command-line + if (zconf.custom_metadata_str) { + json_object *user = + json_tokener_parse(zconf.custom_metadata_str); + if (!user) { + log_error("json-metadata", + "unable to parse user metadata"); + } else { + json_object_object_add(obj, "user-metadata", user); + } + } + + if (zconf.notes) { + json_object_object_add(obj, "notes", + json_object_new_string(zconf.notes)); + } + + // add blocklisted and allowlisted CIDR blocks + bl_cidr_node_t *b = get_blocklisted_cidrs(); + if (b) { + json_object *blocklisted_cidrs = json_object_new_array(); + do { + char cidr[50]; + struct in_addr addr; + addr.s_addr = b->ip_address; + sprintf(cidr, "%s/%i", inet_ntoa(addr), b->prefix_len); + json_object_array_add(blocklisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "blocklisted_networks", + blocklisted_cidrs); + } + + b = get_allowlisted_cidrs(); + if (b) { + json_object *allowlisted_cidrs = json_object_new_array(); + do { + char cidr[50]; + struct in_addr addr; + addr.s_addr = b->ip_address; + sprintf(cidr, "%s/%i", inet_ntoa(addr), b->prefix_len); + json_object_array_add(allowlisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "allowlisted_networks", + allowlisted_cidrs); + } + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + json_object_put(obj); +} diff --git a/rdns_scan/zmap4rdns/src/summary.h b/rdns_scan/zmap4rdns/src/summary.h new file mode 100644 index 0000000..8d1c8a9 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/summary.h @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZMAP_SUMMARY_H +#define ZMAP_SUMMARY_H + +#include <stdio.h> + +void json_metadata(FILE *); + +#endif /* ZMAP_SUMMARY_H */ diff --git a/rdns_scan/zmap4rdns/src/tests/test_harness.c b/rdns_scan/zmap4rdns/src/tests/test_harness.c new file mode 100644 index 0000000..ad4ab0d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/tests/test_harness.c @@ -0,0 +1,97 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <assert.h> +#include <sched.h> +#include <errno.h> +#include <pwd.h> +#include <time.h> +#include <getopt.h> + +#include <pcap/pcap.h> +#include <json.h> +#include <pthread.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#include "aesrand.h" +#include "send.h" +#include "recv.h" +#include "state.h" +#include "monitor.h" +#include "get_gateway.h" +#include "filter.h" +#include "summary.h" + +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" +#include "output_modules/module_json.h" +#include "ztopt.h" + +int test_recursive_fieldsets(void) +{ + fieldset_t *outer = fs_new_fieldset(NULL); + fieldset_t *inner = fs_new_fieldset(NULL); + + fieldset_t *repeated = fs_new_repeated_string(0); + assert(repeated->type == FS_REPEATED); + assert(repeated->len == 0); + assert(repeated->inner_type == FS_STRING); + for (int i = 0; i < 10; i++) { + fs_add_string(repeated, NULL, (char *)"hello world!", 0); + } + fs_add_repeated(outer, (char *)"repeatedstuff", repeated); + fs_add_string(outer, "name", strdup("value"), 0); + fs_add_string(inner, "name2", strdup("value2"), 0); + fs_add_fieldset(outer, "inner", inner); + + print_json_fieldset(outer); + fs_free(outer); + + return EXIT_SUCCESS; +} + +int main(UNUSED int argc, UNUSED char **argv) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + for (int i = 0; i < 100000000; i++) + test_recursive_fieldsets(); + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/topt.ggo.in b/rdns_scan/zmap4rdns/src/topt.ggo.in new file mode 100644 index 0000000..bd02dfa --- /dev/null +++ b/rdns_scan/zmap4rdns/src/topt.ggo.in @@ -0,0 +1,38 @@ +# ZTee Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ztee option description to be processed by gengetopt + +package "ztee" +version "@ZMAP_VERSION@" +purpose "A buffering output splitter" + +section "Basic arguments" + +option "success-only" - "Only write to stdout rows where success=1 or success=true" + optional +option "monitor" m "Print monitor data to stderr" + optional +option "status-updates-file" u "File to write status updates, in CSV format" + typestr="monitor.csv" + optional string +option "log-file" l "File to log errors, etc. to" + optional string +option "raw" r "Ignore input formatting and pass through raw input" + optional + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + zmap -p 80 -o - | ztee zmap.csv (save zmap output to zmap.csv and output all IP addresses to stdout)\n\ + zmap -p 80 --output-fields=* -o - | ztee --success-only zmap.csv (save all zmap output to zmap.csv, print IPs from successful rows to stdout)\n\ + zmap -p 80 -o - | ztee -u status.csv zmap.csv (save zmap output to zmap.csv, write status updates to status.csv, print all IPs to stdout)\n\ + echo \"hello, ztee\" | ztee --raw out.txt (write text to out.txt and to stdout, like tee)" diff --git a/rdns_scan/zmap4rdns/src/topt_compat.c b/rdns_scan/zmap4rdns/src/topt_compat.c new file mode 100644 index 0000000..167ecd5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/topt_compat.c @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "topt.c" diff --git a/rdns_scan/zmap4rdns/src/utility.c b/rdns_scan/zmap4rdns/src/utility.c new file mode 100644 index 0000000..ecd320f --- /dev/null +++ b/rdns_scan/zmap4rdns/src/utility.c @@ -0,0 +1,71 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "utility.h" + +#include <stdio.h> +#include <arpa/inet.h> + +#include "state.h" +#include "../lib/logger.h" + +in_addr_t string_to_ip_address(char *t) +{ + in_addr_t r = inet_addr(t); + if (r == INADDR_NONE) { + log_fatal("send", "invalid ip address: `%s'", t); + } + return r; +} + +void add_to_array(char *to_add) +{ + if (zconf.number_source_ips >= 256) { + // log fatal here + log_fatal("parse", "over 256 source IP addresses provided"); + } + log_debug("SEND", "ipaddress: %s\n", to_add); + zconf.source_ip_addresses[zconf.number_source_ips] = + string_to_ip_address(to_add); + zconf.number_source_ips++; +} + +void parse_source_ip_addresses(char given_string[]) +{ + char *dash = strchr(given_string, '-'); + char *comma = strchr(given_string, ','); + if (dash && comma) { + *comma = '\0'; + parse_source_ip_addresses(given_string); + parse_source_ip_addresses(comma + 1); + } else if (comma) { + while (comma) { + *comma = '\0'; + add_to_array(given_string); + given_string = comma + 1; + comma = strchr(given_string, ','); + if (!comma) { + add_to_array(given_string); + } + } + } else if (dash) { + *dash = '\0'; + log_debug("SEND", "address: %s\n", given_string); + log_debug("SEND", "address: %s\n", dash + 1); + in_addr_t start = ntohl(string_to_ip_address(given_string)); + in_addr_t end = ntohl(string_to_ip_address(dash + 1)) + 1; + while (start != end) { + struct in_addr temp; + temp.s_addr = htonl(start); + add_to_array(strdup(inet_ntoa(temp))); + start++; + } + } else { + add_to_array(given_string); + } +} diff --git a/rdns_scan/zmap4rdns/src/utility.h b/rdns_scan/zmap4rdns/src/utility.h new file mode 100644 index 0000000..dbf0258 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/utility.h @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UTILITY_H +#define UTILITY_H + +void parse_source_ip_addresses(char given_string[]); + +#endif // UTILITY_H diff --git a/rdns_scan/zmap4rdns/src/validate.c b/rdns_scan/zmap4rdns/src/validate.c new file mode 100644 index 0000000..75f9c64 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/validate.c @@ -0,0 +1,53 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <stdint.h> +#include <assert.h> +#include "../lib/rijndael-alg-fst.h" +#include "../lib/random.h" +#include "../lib/logger.h" +#include "validate.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 + +static int inited = 0; +static uint32_t aes_sched[(AES_ROUNDS + 1) * 4]; + +void validate_init() +{ + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("validate", "couldn't get random bytes"); + } + if (rijndaelKeySetupEnc(aes_sched, key, AES_KEY_BYTES * 8) != AES_ROUNDS) { + log_fatal("validate", "couldn't initialize AES key"); + } + inited = 1; +} + +void validate_gen(const uint32_t src, const uint32_t dst, + uint8_t output[VALIDATE_BYTES]) +{ + validate_gen_ex(src, dst, 0, 0, output); +} + +void validate_gen_ex(const uint32_t input0, const uint32_t input1, + const uint32_t input2, const uint32_t input3, + uint8_t output[VALIDATE_BYTES]) +{ + assert(inited); + + uint32_t aes_input[AES_BLOCK_WORDS]; + aes_input[0] = input0; + aes_input[1] = input1; + aes_input[2] = input2; + aes_input[3] = input3; + rijndaelEncrypt(aes_sched, AES_ROUNDS, (uint8_t *)aes_input, output); +} diff --git a/rdns_scan/zmap4rdns/src/validate.h b/rdns_scan/zmap4rdns/src/validate.h new file mode 100644 index 0000000..0ed8078 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/validate.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef VALIDATE_H +#define VALIDATE_H + +#define VALIDATE_BYTES 16 + +void validate_init(); +void validate_gen(const uint32_t src, const uint32_t dst, + uint8_t output[VALIDATE_BYTES]); +void validate_gen_ex(const uint32_t input0, const uint32_t input1, + const uint32_t input2, const uint32_t input3, + uint8_t output[VALIDATE_BYTES]); + +#endif //_VALIDATE_H diff --git a/rdns_scan/zmap4rdns/src/zblocklist.1 b/rdns_scan/zmap4rdns/src/zblocklist.1 new file mode 100644 index 0000000..db85a47 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zblocklist.1 @@ -0,0 +1,60 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZBLACKLIST" "1" "June 2017" "ZMap" "zblocklist" +. +.SH "NAME" +\fBzblocklist\fR \- zmap IP blacklist tool +. +.SH "SYNOPSIS" +zblocklist [ \-b <blacklist> ] [ \-w <allowlist> ] [ OPTIONS\.\.\. ] +. +.SH "DESCRIPTION" +\fIZBlacklist\fR is a network tool for limiting and deduplicating a list of IP addresses using a blocklist or allowlist\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblacklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=name\fR +File of subnets to include, in CIDR notation, one\-per line\. All other subnets will be excluded\. +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +File to log to\. +. +.TP +\fB\-\-disable\-syslog\fR +Disable logging messages to syslog\. +. +.TP +\fB\-v\fR, \fB\-\-verbosity\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-\-no\-duplicate\-checking\fR +Don\'t deduplicate input addresses\. Default is false\. +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid, malformed, or unresolvable entries in the blocklist/allowlist\. Default is false\. +. +.TP +\fB\-\-ignore\-input\-errors\fR +Don\'t print invalid entries in the input\. Default is false\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help and exit +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit + diff --git a/rdns_scan/zmap4rdns/src/zblocklist.1.ronn b/rdns_scan/zmap4rdns/src/zblocklist.1.ronn new file mode 100644 index 0000000..c2e8fb5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zblocklist.1.ronn @@ -0,0 +1,53 @@ +zblocklist(1) - zmap IP blocklist tool +====================================== + +## SYNOPSIS + +zblocklist [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ] + +## DESCRIPTION + +*ZBlacklist* is a network tool for limiting and deduplicating a list of +IP addresses using a blocklist or allowlist. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=name`: + File of subnets to include, in CIDR notation, one-per line. All other + subnets will be excluded. + + * `-l`, `--log-file=name`: + File to log to. + + * `--disable-syslog`: + Disable logging messages to syslog. + + * `-v`, `--verbosity`: + Level of log detail (0-5, default=3) + + * `--no-duplicate-checking`: + Don't deduplicate input addresses. Default is false. + + * `--ignore-blocklist-errors`: + Ignore invalid, malformed, or unresolvable entries in the + blocklist/allowlist. Default is false. + + * `--ignore-input-errors`: + Don't print invalid entries in the input. Default is false. + + +### ADDITIONAL OPTIONS ### + + * `-h`, `--help`: + Print help and exit + + * `-V`, `--version`: + Print version and exit diff --git a/rdns_scan/zmap4rdns/src/zblocklist.c b/rdns_scan/zmap4rdns/src/zblocklist.c new file mode 100644 index 0000000..ec9da18 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zblocklist.c @@ -0,0 +1,263 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * ZBlocklist is a simple utility that (1) excludes IP addresses on a specified + * blocklist from being scanned, and (2) ensures the uniqueness of output + * addresses such that no host is scanned twice. ZBlocklist takes in a list + * of addresses on stdin and outputs addresses that are acceptable to scan + * on stdout. The utility uses the blocklist data structures from ZMap for + * checking scan eligibility and a paged bitmap for duplicate prevention. + */ + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <getopt.h> +#include <assert.h> +#include <sched.h> +#include <errno.h> +#include <pwd.h> +#include <time.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/pbm.h" + +#include "zbopt.h" + +// struct zbl_stats { +// uint32_t cidr_entries; +// uint32_t allowed_addrs; +// uint32_t input_addrs; +// uint32_t uniq_input_addrs; +// uint32_t blocked_addrs; +// uint32_t output_addrs; +// uint32_t duplicates; +//}; + +#undef MIN +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) + +// allow 1mb lines + newline + \0 +#define MAX_LINE_LENGTH 1024 * 1024 + 2 + +static inline char *zmin(char *a, char *b) +{ + if (a && !b) + return a; + else if (b && !a) + return b; + else + return MIN(a, b); +} + +struct zbl_conf { + char *blocklist_filename; + char *allowlist_filename; + char *log_filename; + int check_duplicates; + int ignore_blocklist_errors; + int ignore_input_errors; + int verbosity; + int disable_syslog; + // struct zbl_stats stats; +}; + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char **argv) +{ + struct zbl_conf conf; + conf.verbosity = 3; + memset(&conf, 0, sizeof(struct zbl_conf)); + int no_dupchk_pres = 0; + conf.ignore_blocklist_errors = 0; + conf.ignore_input_errors = 0; + + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Set the log file and metadata file + if (args.log_file_given) { + conf.log_filename = strdup(args.log_file_arg); + } + if (args.verbosity_given) { + conf.verbosity = args.verbosity_arg; + } + + // Blocklist and allowlist + if (args.blocklist_file_given) { + conf.blocklist_filename = strdup(args.blocklist_file_arg); + } + if (args.allowlist_file_given) { + conf.allowlist_filename = strdup(args.allowlist_file_arg); + } + + // Read the boolean flags + SET_BOOL(no_dupchk_pres, no_duplicate_checking); + conf.check_duplicates = !no_dupchk_pres; + SET_BOOL(conf.ignore_blocklist_errors, ignore_blocklist_errors); + SET_BOOL(conf.ignore_input_errors, ignore_input_errors); + SET_BOOL(conf.disable_syslog, disable_syslog); + + // initialize logging + FILE *logfile = stderr; + if (conf.log_filename) { + logfile = fopen(conf.log_filename, "w"); + if (!logfile) { + fprintf( + stderr, + "FATAL: unable to open specified logfile (%s)\n", + conf.log_filename); + exit(1); + } + } + if (log_init(logfile, conf.verbosity, !conf.disable_syslog, + "zblocklist")) { + fprintf(stderr, "FATAL: unable able to initialize logging\n"); + exit(1); + } + + if (!conf.blocklist_filename && !conf.allowlist_filename) { + log_fatal("zblocklist", + "must specify either a allowlist or blocklist file"); + } + + // parse blocklist + if (conf.blocklist_filename) { + log_debug("zblocklist", "blocklist file at %s to be used", + conf.blocklist_filename); + } else { + log_debug("zblocklist", "no blocklist file specified"); + } + if (conf.blocklist_filename && + access(conf.blocklist_filename, R_OK) == -1) { + log_fatal("zblocklist", + "unable to read specified blocklist file (%s)", + conf.blocklist_filename); + } + if (conf.allowlist_filename) { + log_debug("zblocklist", "allowlist file at %s to be used", + conf.allowlist_filename); + } else { + log_debug("zblocklist", "no allowlist file specified"); + } + if (conf.allowlist_filename && + access(conf.allowlist_filename, R_OK) == -1) { + log_fatal("zblocklist", + "unable to read specified allowlist file (%s)", + conf.allowlist_filename); + } + + if (blocklist_init(conf.allowlist_filename, conf.blocklist_filename, + NULL, 0, NULL, 0, conf.ignore_blocklist_errors)) { + log_fatal("zmap", "unable to initialize blocklist / allowlist"); + } + // initialize paged bitmap + uint8_t **seen = NULL; + if (conf.check_duplicates) { + seen = pbm_init(); + if (!seen) { + log_fatal("zblocklist", + "unable to initialize paged bitmap"); + } + } + // process addresses + char *line = malloc(MAX_LINE_LENGTH); + assert(line); + char *original = malloc(MAX_LINE_LENGTH); + assert(original); + while (fgets(line, MAX_LINE_LENGTH, stdin) != NULL) { + size_t len = strlen(line); + if (len >= (MAX_LINE_LENGTH - 1)) { + log_fatal("zblocklist", + "received line longer than max length: %i", + MAX_LINE_LENGTH); + } + // remove new line + memcpy(original, line, len + 1); + char *n = + zmin(zmin(zmin(zmin(strchr(line, '\n'), strchr(line, ',')), + strchr(line, '\t')), + strchr(line, ' ')), + strchr(line, '#')); + assert(n); + n[0] = 0; + log_debug("zblocklist", "input value %s", line); + // parse into int + struct in_addr addr; + if (!inet_aton(line, &addr)) { + log_warn("zblocklist", "invalid input address: %s", + line); + if (!conf.ignore_input_errors) { + printf("%s", original); + } + continue; + } + if (conf.check_duplicates) { + if (pbm_check(seen, ntohl(addr.s_addr))) { + log_debug("zblocklist", + "%s is a duplicate: skipped", line); + continue; + } else { + log_debug("zblocklist", + "%s not a duplicate: skipped", line); + } + } else { + log_debug("zblocklist", "no duplicate checking for %s", + line); + } + // check if in blocklist + if (blocklist_is_allowed(addr.s_addr)) { + if (conf.check_duplicates) { + if (!pbm_check(seen, ntohl(addr.s_addr))) { + pbm_set(seen, ntohl(addr.s_addr)); + printf("%s", original); + } + } else { + printf("%s", original); + } + } + } + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/zbopt.ggo.in b/rdns_scan/zmap4rdns/src/zbopt.ggo.in new file mode 100644 index 0000000..46359f2 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zbopt.ggo.in @@ -0,0 +1,43 @@ +# ZBlacklist Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# zblocklist option description to be processed by gengetopt + +package "zblocklist" +version "@ZMAP_VERSION@" +purpose "A tool for limiting and deduplicating a list of IP addresses" + +section "Basic arguments" + +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, one-per line." + optional string +option "allowlist-file" w "File of subnets to include, in CIDR notation, one-per line." + optional string +option "log-file" l "File to log to" + optional string +option "verbosity" v "Set log level verbosity (0-5, default 3)" + default="3" + optional int +option "no-duplicate-checking" - "Don't deduplicate IP addresses (default false)" + optional +option "ignore-blocklist-errors" - "Ignore invalid entries in the blocklist/allowlist (default false)" + optional +option "ignore-input-errors" - "Don't print invalid entries in the input (default false)" + optional +option "disable-syslog" - "Disables logging messages to syslog" + optional + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +section "Notes" + +text + "At least one of --allowlist-file or --blocklist-file must be specified. Blacklist files take precedence over allowlist files when both are specified. This results in an output of {allowlist - blocklist}." diff --git a/rdns_scan/zmap4rdns/src/zbopt_compat.c b/rdns_scan/zmap4rdns/src/zbopt_compat.c new file mode 100644 index 0000000..5928c1a --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zbopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zbopt.c" diff --git a/rdns_scan/zmap4rdns/src/ziterate.1 b/rdns_scan/zmap4rdns/src/ziterate.1 new file mode 100644 index 0000000..78ca761 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ziterate.1 @@ -0,0 +1,70 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZITERATE" "1" "June 2017" "ZMap" "ziterate" +. +.SH "NAME" +\fBziterate\fR \- ZMap IP permutation generation file +. +.SH "SYNOPSIS" +ziterate [ \-b <blocklist> ] [ \-w <allowlist> ] [ OPTIONS\.\.\. ] +. +.SH "DESCRIPTION" +\fIZIterate\fR is a network tool that will produce IPv4 addresses in a psuedorandom order similar to how ZMap generates random addresses to be scanned\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblacklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=name\fR +File of subnets to include, in CIDR notation, one\-per line\. All other subnets will be excluded\. +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +File to log to\. +. +.TP +\fB\-\-disable\-syslog\fR +Disable logging messages to syslog\. +. +.TP +\fB\-v\fR, \fB\-\-verbosity\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid entries in the blocklist\. Default is false\. +. +.TP +\fB\-\-seed=n\fR +Seed used to select address permutation\. +. +.TP +\fB\-n\fR, \fB\-\-max\-targets=n\fR +Cap number of IPs to generate (as a number or a percentage of the address space) +. +.SS "SHARDING" +. +.TP +\fB\-\-shards=n\fR +Total number of shards\. +. +.TP +\fB\-\-shard=n\fR +Shard this scan is targeting\. Zero indexed\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help text and exit\. +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit\. + diff --git a/rdns_scan/zmap4rdns/src/ziterate.1.html b/rdns_scan/zmap4rdns/src/ziterate.1.html new file mode 100644 index 0000000..32a77a0 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ziterate.1.html @@ -0,0 +1,127 @@ +<!DOCTYPE html> +<html> +<head> + <meta http-equiv='content-type' value='text/html;charset=utf8'> + <meta name='generator' value='Ronn/v0.7.3 (http://github.com/rtomayko/ronn/tree/0.7.3)'> + <title>ziterate(1) - ZMap IP permutation generation file</title> + <style type='text/css' media='all'> + /* style: man */ + body#manpage {margin:0} + .mp {max-width:100ex;padding:0 9ex 1ex 4ex} + .mp p,.mp pre,.mp ul,.mp ol,.mp dl {margin:0 0 20px 0} + .mp h2 {margin:10px 0 0 0} + .mp > p,.mp > pre,.mp > ul,.mp > ol,.mp > dl {margin-left:8ex} + .mp h3 {margin:0 0 0 4ex} + .mp dt {margin:0;clear:left} + .mp dt.flush {float:left;width:8ex} + .mp dd {margin:0 0 0 9ex} + .mp h1,.mp h2,.mp h3,.mp h4 {clear:left} + .mp pre {margin-bottom:20px} + .mp pre+h2,.mp pre+h3 {margin-top:22px} + .mp h2+pre,.mp h3+pre {margin-top:5px} + .mp img {display:block;margin:auto} + .mp h1.man-title {display:none} + .mp,.mp code,.mp pre,.mp tt,.mp kbd,.mp samp,.mp h3,.mp h4 {font-family:monospace;font-size:14px;line-height:1.42857142857143} + .mp h2 {font-size:16px;line-height:1.25} + .mp h1 {font-size:20px;line-height:2} + .mp {text-align:justify;background:#fff} + .mp,.mp code,.mp pre,.mp pre code,.mp tt,.mp kbd,.mp samp {color:#131211} + .mp h1,.mp h2,.mp h3,.mp h4 {color:#030201} + .mp u {text-decoration:underline} + .mp code,.mp strong,.mp b {font-weight:bold;color:#131211} + .mp em,.mp var {font-style:italic;color:#232221;text-decoration:none} + .mp a,.mp a:link,.mp a:hover,.mp a code,.mp a pre,.mp a tt,.mp a kbd,.mp a samp {color:#0000ff} + .mp b.man-ref {font-weight:normal;color:#434241} + .mp pre {padding:0 4ex} + .mp pre code {font-weight:normal;color:#434241} + .mp h2+pre,h3+pre {padding-left:0} + ol.man-decor,ol.man-decor li {margin:3px 0 10px 0;padding:0;float:left;width:33%;list-style-type:none;text-transform:uppercase;color:#999;letter-spacing:1px} + ol.man-decor {width:100%} + ol.man-decor li.tl {text-align:left} + ol.man-decor li.tc {text-align:center;letter-spacing:4px} + ol.man-decor li.tr {text-align:right;float:right} + </style> +</head> +<!-- + The following styles are deprecated and will be removed at some point: + div#man, div#man ol.man, div#man ol.head, div#man ol.man. + + The .man-page, .man-decor, .man-head, .man-foot, .man-title, and + .man-navigation should be used instead. +--> +<body id='manpage'> + <div class='mp' id='man'> + + <div class='man-navigation' style='display:none'> + <a href="#NAME">NAME</a> + <a href="#SYNOPSIS">SYNOPSIS</a> + <a href="#DESCRIPTION">DESCRIPTION</a> + <a href="#OPTIONS">OPTIONS</a> + </div> + + <ol class='man-decor man-head man head'> + <li class='tl'>ziterate(1)</li> + <li class='tc'>ziterate</li> + <li class='tr'>ziterate(1)</li> + </ol> + + <h2 id="NAME">NAME</h2> +<p class="man-name"> + <code>ziterate</code> - <span class="man-whatis">ZMap IP permutation generation file</span> +</p> + +<h2 id="SYNOPSIS">SYNOPSIS</h2> + +<p>ziterate [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ]</p> + +<h2 id="DESCRIPTION">DESCRIPTION</h2> + +<p><em>ZIterate</em> is a network tool that will produce IPv4 addresses in a psuedorandom +order similar to how ZMap generates random addresses to be scanned.</p> + +<h2 id="OPTIONS">OPTIONS</h2> + +<h3 id="BASIC-OPTIONS">BASIC OPTIONS</h3> + +<dl> +<dt> <code>-b</code>, <code>--blocklist-file=path</code></dt><dd><p> File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file <strong>blocklist.conf</strong> for this purpose.</p></dd> +<dt><code>-w</code>, <code>--allowlist-file=name</code></dt><dd><p>File of subnets to include, in CIDR notation, one-per line. All other +subnets will be excluded.</p></dd> +<dt><code>-l</code>, <code>--log-file=name</code></dt><dd><p>File to log to.</p></dd> +<dt><code>--disable-syslog</code></dt><dd><p>Disable logging messages to syslog.</p></dd> +<dt><code>-v</code>, <code>--verbosity</code></dt><dd><p>Level of log detail (0-5, default=3)</p></dd> +<dt><code>--ignore-blocklist-errors</code></dt><dd><p>Ignore invalid entries in the blocklist. Default is false.</p></dd> +<dt><code>--seed=n</code></dt><dd><p>Seed used to select address permutation.</p></dd> +<dt><code>-n</code>, <code>--max-targets=n</code></dt><dd><p>Cap number of IPs to generate (as a number or a percentage of the address space)</p></dd> +</dl> + + +<h3 id="SHARDING">SHARDING</h3> + +<dl> +<dt><code>--shards=n</code></dt><dd><p>Total number of shards.</p></dd> +<dt><code>--shard=n</code></dt><dd><p>Shard this scan is targeting. Zero indexed.</p></dd> +</dl> + + +<h3 id="ADDITIONAL-OPTIONS">ADDITIONAL OPTIONS</h3> + +<dl> +<dt><code>-h</code>, <code>--help</code></dt><dd><p>Print help text and exit.</p></dd> +<dt><code>-V</code>, <code>--version</code></dt><dd><p>Print version and exit.</p></dd> +</dl> + + + + <ol class='man-decor man-foot man foot'> + <li class='tl'>ZMap</li> + <li class='tc'>June 2017</li> + <li class='tr'>ziterate(1)</li> + </ol> + + </div> +</body> +</html> diff --git a/rdns_scan/zmap4rdns/src/ziterate.1.ronn b/rdns_scan/zmap4rdns/src/ziterate.1.ronn new file mode 100644 index 0000000..0282ee0 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ziterate.1.ronn @@ -0,0 +1,61 @@ +ziterate(1) - ZMap IP permutation generation file +================================================= + +## SYNOPSIS + +ziterate [ -b <blocklist> ] [ -w <allowlist> ] [ OPTIONS... ] + +## DESCRIPTION + +*ZIterate* is a network tool that will produce IPv4 addresses in a psuedorandom +order similar to how ZMap generates random addresses to be scanned. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=name`: + File of subnets to include, in CIDR notation, one-per line. All other + subnets will be excluded. + + * `-l`, `--log-file=name`: + File to log to. + + * `--disable-syslog`: + Disable logging messages to syslog. + + * `-v`, `--verbosity`: + Level of log detail (0-5, default=3) + + * `--ignore-blocklist-errors`: + Ignore invalid entries in the blocklist. Default is false. + + * `--seed=n`: + Seed used to select address permutation. + + * `-n`, `--max-targets=n`: + Cap number of IPs to generate (as a number or a percentage of the address space) + + +### SHARDING ### + + * `--shards=n`: + Total number of shards. + + * `--shard=n`: + Shard this scan is targeting. Zero indexed. + + +### ADDITIONAL OPTIONS ### + + * `-h`, `--help`: + Print help text and exit. + + * `-V`, `--version`: + Print version and exit. diff --git a/rdns_scan/zmap4rdns/src/ziterate.c b/rdns_scan/zmap4rdns/src/ziterate.c new file mode 100644 index 0000000..c0af2b5 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ziterate.c @@ -0,0 +1,226 @@ +/* + * ZMap Copyright 2016 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* + * ZIterate is a simple utility that will iteratate over the IPv4 + * space in a pseudo-random fashion, utilizing the sharding capabilities * of + * ZMap. + */ + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <getopt.h> +#include <assert.h> +#include <unistd.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" + +#include "iterator.h" +#include "state.h" +#include "validate.h" +#include "zitopt.h" + +struct zit_conf { + char *blocklist_filename; + char *allowlist_filename; + char **destination_cidrs; + int destination_cidrs_len; + char *log_filename; + int check_duplicates; + int ignore_errors; + int verbosity; + int disable_syslog; + + // sharding options + uint16_t shard_num; + uint16_t total_shards; + uint64_t seed; + aesrand_t *aes; + uint32_t max_hosts; +}; + +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char **argv) +{ + struct zit_conf conf; + + memset(&conf, 0, sizeof(struct zit_conf)); + conf.verbosity = 3; + conf.ignore_errors = 0; + + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Set the log file and metadata file + if (args.log_file_given) { + conf.log_filename = strdup(args.log_file_arg); + } + if (args.verbosity_given) { + conf.verbosity = args.verbosity_arg; + } + // Read the boolean flags + SET_BOOL(conf.ignore_errors, ignore_blocklist_errors); + SET_BOOL(conf.disable_syslog, disable_syslog); + + // initialize logging + FILE *logfile = stderr; + if (conf.log_filename) { + logfile = fopen(conf.log_filename, "w"); + if (!logfile) { + fprintf( + stderr, + "FATAL: unable to open specified logfile (%s)\n", + conf.log_filename); + exit(1); + } + } + if (log_init(logfile, conf.verbosity, !conf.disable_syslog, + "ziterate")) { + fprintf(stderr, "FATAL: unable able to initialize logging\n"); + exit(1); + } + + // Blocklist and allowlist + if (args.blocklist_file_given) { + conf.blocklist_filename = strdup(args.blocklist_file_arg); + } + if (args.allowlist_file_given) { + conf.allowlist_filename = strdup(args.allowlist_file_arg); + } + conf.destination_cidrs = args.inputs; + conf.destination_cidrs_len = args.inputs_num; + // max targets + if (args.max_targets_given) { + conf.max_hosts = parse_max_hosts(args.max_targets_arg); + } + + // sanity check blocklist file + if (conf.blocklist_filename) { + log_debug("ziterate", "blocklist file at %s to be used", + conf.blocklist_filename); + } else { + log_debug("ziterate", "no blocklist file specified"); + } + if (conf.blocklist_filename && + access(conf.blocklist_filename, R_OK) == -1) { + log_fatal("ziterate", + "unable to read specified blocklist file (%s)", + conf.blocklist_filename); + } + + // sanity check allowlist file + if (conf.allowlist_filename) { + log_debug("ziterate", "allowlist file at %s to be used", + conf.allowlist_filename); + } else { + log_debug("ziterate", "no allowlist file specified"); + } + if (conf.allowlist_filename && + access(conf.allowlist_filename, R_OK) == -1) { + log_fatal("ziterate", + "unable to read specified allowlist file (%s)", + conf.allowlist_filename); + } + + // parse blocklist and allowlist + if (blocklist_init(conf.allowlist_filename, conf.blocklist_filename, + conf.destination_cidrs, conf.destination_cidrs_len, + NULL, 0, conf.ignore_errors)) { + log_fatal("ziterate", + "unable to initialize blocklist / allowlist"); + } + + // Set up sharding + conf.shard_num = 0; + conf.total_shards = 1; + if ((args.shard_given || args.shards_given) && !args.seed_given) { + log_fatal("ziterate", + "Need to specify seed if sharding a scan"); + } + if (args.shard_given ^ args.shards_given) { + log_fatal( + "ziterate", + "Need to specify both shard number and total number of shards"); + } + if (args.shard_given) { + enforce_range("shard", args.shard_arg, 0, 65534); + conf.shard_num = args.shard_arg; + } + if (args.shards_given) { + enforce_range("shards", args.shards_arg, 1, 65535); + conf.total_shards = args.shards_arg; + } + if (conf.shard_num >= conf.total_shards) { + log_fatal("ziterate", + "With %hhu total shards, shard number (%hhu)" + " must be in range [0, %hhu)", + conf.total_shards, conf.shard_num, conf.total_shards); + } + log_debug( + "ziterate", + "Initializing sharding (%d shards, shard number %d, seed %llu)", + conf.total_shards, conf.shard_num, conf.seed); + + // Check for a random seed + if (args.seed_given) { + conf.seed = args.seed_arg; + } else { + if (!random_bytes(&conf.seed, sizeof(uint64_t))) { + log_fatal("ziterate", "unable to generate random bytes " + "needed for seed"); + } + } + zconf.aes = aesrand_init_from_seed(conf.seed); + + iterator_t *it = iterator_init(1, conf.shard_num, conf.total_shards); + shard_t *shard = get_shard(it, 0); + uint32_t next_int = shard_get_cur_ip(shard); + struct in_addr next_ip; + + for (uint32_t count = 0; next_int; ++count) { + if (conf.max_hosts && count >= conf.max_hosts) { + break; + } + next_ip.s_addr = next_int; + printf("%s\n", inet_ntoa(next_ip)); + next_int = shard_get_next_ip(shard); + } + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/zitopt.ggo.in b/rdns_scan/zmap4rdns/src/zitopt.ggo.in new file mode 100644 index 0000000..175700f --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zitopt.ggo.in @@ -0,0 +1,55 @@ +# ZIterate Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ziterate option description to be processed by gengetopt + +package "ziterate" +version "@ZMAP_VERSION@" +purpose "A tool for iterating over the IPv4 space" + +section "Basic arguments" + +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, one-per line." + optional string +option "allowlist-file" w "File of subnets to include, in CIDR notation, one-per line." + optional string +option "log-file" l "File to log to" + optional string +option "verbosity" v "Set log level verbosity (0-5, default 3)" + default="3" + optional int +option "ignore-blocklist-errors" - "Ignore invalid entries in the blocklist/allowlist (default false)" + optional +option "seed" e "Seed used to select address permutation" + typestr="n" + optional longlong +option "max-targets" n "Cap number of IPs to generate (as a number or a percentage of the address space)" + typestr="n" + optional string +option "disable-syslog" - "Disables logging messages to syslog" + optional + +section "Sharding" + +option "shards" - "total number of shards" + typestr="N" + optional int + default="1" +option "shard" - "shard this scan is targeting (0 indexed)" + typestr="n" + optional int + default="0" + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + ziterate (iterate over all public IPv4 addresses)\n\ + ziterate -b exclusions 10.0.0.0/8 (iterate all IPs in 10./8 except those in blocklist)\n" diff --git a/rdns_scan/zmap4rdns/src/zitopt_compat.c b/rdns_scan/zmap4rdns/src/zitopt_compat.c new file mode 100644 index 0000000..5ba6275 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zitopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zitopt.c" diff --git a/rdns_scan/zmap4rdns/src/zmap.1 b/rdns_scan/zmap4rdns/src/zmap.1 new file mode 100644 index 0000000..2725c13 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zmap.1 @@ -0,0 +1,269 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZMAP" "1" "June 2017" "ZMap" "zmap" +. +.SH "NAME" +\fBzmap\fR \- The Fast Internet Scanner +. +.SH "SYNOPSIS" +zmap [ \-p <port> ] [ \-o <outfile> ] [ OPTIONS\.\.\. ] [ ip/hostname/range ] +. +.SH "DESCRIPTION" +\fIZMap\fR is a network tool for scanning the entire IPv4 address space (or large samples)\. ZMap is capable of scanning the entire Internet in around 45 minutes on a gigabit network connection, reaching ~98% theoretical line speed\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fBip\fR/\fBhostname\fR/\fBrange\fR +IP addresses or DNS hostnames to scan\. Accepts IP ranges in CIDR block notation\. Defaults to 0\.0\.0/8 +. +.TP +\fB\-p\fR, \fB\-\-target\-port=port\fR +TCP or UDP port number to scan (for SYN scans and basic UDP scans) +. +.TP +\fB\-o\fR, \fB\-\-output\-file=name\fR +When using an output module that uses a file, write results to this file\. Use \- for stdout\. +. +.TP +\fB\-b\fR, \fB\-\-blocklist\-file=path\fR +File of subnets to exclude, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blocklist file \fBblocklist\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-allowlist\-file=path\fR +File of subnets to scan, in CIDR notation, one\-per line\. Specifying a allowlist file is equivalent to specifying to ranges directly on the command line interface, but allows specifying a large number of subnets\. Note: if you are specifying a large number of individual IP addresses (more than 10 million), you should instead use \fB\-\-list\-of\-ips\-file\fR\. +. +.TP +\fB\-I\fR, \fB\-\-list\-of\-ips\-file=path\fR +File of individual IP addresses to scan, one\-per line\. This feature allows you to scan a large number of unrelated addresses\. If you have a small number of IPs, it is faster to specify these on the command line or by using \fB\-\-allowlist\-file\fR\. This should only be used when scanning more than 10 million addresses\. When used in with \-\-allowlist\-path, only hosts in the intersection of both sets will be scanned\. Hosts specified here, but included in the blocklist will be excluded\. +. +.SS "SCAN OPTIONS" +. +.TP +\fB\-r\fR, \fB\-\-rate=pps\fR +Set the send rate in packets/sec\. Note: when combined with \-\-probes, this is total packets per second, not IPs per second\. +. +.TP +\fB\-B\fR, \fB\-\-bandwidth=bps\fR +Set the send rate in bits/second (supports suffixes G, M, and K (e\.g\. \-B 10M for 10 mbps)\. This overrides the \-\-rate flag\. +. +.TP +\fB\-n\fR, \fB\-\-max\-targets=n\fR +Cap the number of targets to probe\. This can either be a number (e\.g\. \-n 1000) or a percentage (e\.g\. \-n 0\.1%) of the scannable address space (after excluding blocklist) +. +.TP +\fB\-N\fR, \fB\-\-max\-results=n\fR +Exit after receiving this many results +. +.TP +\fB\-t\fR, \fB\-\-max\-runtime=secs\fR +Cap the length of time for sending packets +. +.TP +\fB\-c\fR, \fB\-\-cooldown\-time=secs\fR +How long to continue receiving after sending has completed (default=8) +. +.TP +\fB\-e\fR, \fB\-\-seed=n\fR +Seed used to select address permutation\. Use this if you want to scan addresses in the same order for multiple ZMap runs\. +. +.TP +\fB\-\-shards=N\fR +Split the scan up into N shards/partitions among different instances of zmap (default=1)\. When sharding, \fB\-\-seed\fR is required\. +. +.TP +\fB\-\-shard=n\fR +Set which shard to scan (default=0)\. Shards are 0\-indexed in the range [0, N), where N is the total number of shards\. When sharding \fB\-\-seed\fR is required\. +. +.TP +\fB\-P\fR, \fB\-\-probes=n\fR +Number of probes to send to each IP (default=1) +. +.TP +\fB\-\-retries=n\fR +Number of times to try resending a packet if the sendto call fails (default=10) +. +.SS "NETWORK OPTIONS" +. +.TP +\fB\-s\fR, \fB\-\-source\-port=port|range\fR +Source port(s) to send packets from +. +.TP +\fB\-S\fR, \fB\-\-source\-ip=ip|range\fR +Source address(es) to send packets from\. Either single IP or range (e\.g\. 10\.0\.0\.1\-10\.0\.0\.9) +. +.TP +\fB\-G\fR, \fB\-\-gateway\-mac=addr\fR +Gateway MAC address to send packets to (in case auto\-detection fails) +. +.TP +\fB\-\-source\-mac=addr\fR +Source MAC address to send packets from (in case auto\-detection fails) +. +.TP +\fB\-i\fR, \fB\-\-interface=name\fR +Network interface to use +. +.TP +\fB\-X\fR, \fB\-\-iplayer\fR +Send IP layer packets instead of ethernet packets (for non\-Ethernet interface) +. +.SS "PROBE OPTIONS" +ZMap allows users to specify and write their own probe modules\. Probe modules are responsible for generating probe packets to send, and processing responses from hosts\. +. +.TP +\fB\-\-list\-probe\-modules\fR +List available probe modules (e\.g\. tcp_synscan) +. +.TP +\fB\-M\fR, \fB\-\-probe\-module=name\fR +Select probe module (default=tcp_synscan) +. +.TP +\fB\-\-probe\-args=args\fR +Arguments to pass to probe module +. +.TP +\fB\-\-list\-output\-fields\fR +List the fields the selected probe module can send to the output module +. +.SS "OUTPUT OPTIONS" +ZMap allows users to specify and write their own output modules for use with ZMap\. Output modules are responsible for processing the fieldsets returned by the probe module, and outputting them to the user\. Users can specify output fields, and write filters over the output fields\. +. +.TP +\fB\-\-list\-output\-modules\fR +List available output modules (e\.g\. csv) +. +.TP +\fB\-O\fR, \fB\-\-output\-module=name\fR +Select output module (default=csv) +. +.TP +\fB\-\-output\-args=args\fR +Arguments to pass to output module +. +.TP +\fB\-f\fR, \fB\-\-output\-fields=fields\fR +Comma\-separated list of fields to output +. +.TP +\fB\-\-output\-filter\fR +Specify an output filter over the fields defined by the probe module\. See the output filter section for more details\. +. +.SS "LOGGING AND METADATA OPTIONS" +. +.TP +\fB\-q\fR, \fB\-\-quiet\fR +Do not print status updates once per second +. +.TP +\fB\-v\fR, \fB\-\-verbosity=n\fR +Level of log detail (0\-5, default=3) +. +.TP +\fB\-l\fR, \fB\-\-log\-file=filename\fR +Output file for log messages\. By default, stderr\. +. +.TP +\fB\-m\fR, \fB\-\-metadata\-file=filename\fR +Output file for scan metadata (JSON) +. +.TP +\fB\-L\fR, \fB\-\-log\-directory\fR +Write log entries to a timestamped file in this directory +. +.TP +\fB\-u\fR, \fB\-\-status\-updates\-file\fR +Write scan progress updates to CSV file" +. +.TP +\fB\-\-disable\-syslog\fR +Disables logging messages to syslog +. +.TP +\fB\-\-notes\fR +Inject user\-specified notes into scan metadata +. +.TP +\fB\-\-user\-metadata\fR +Inject user\-specified JSON metadata into scan metadata +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-T\fR, \fB\-\-sender\-threads=n\fR +Threads used to send packets\. ZMap will attempt to detect the optimal number of send threads based on the number of processor cores\. +. +.TP +\fB\-C\fR, \fB\-\-config=filename\fR +Read a configuration file, which can specify any other options\. +. +.TP +\fB\-d\fR, \fB\-\-dryrun\fR +Print out each packet to stdout instead of sending it (useful for debugging) +. +.TP +\fB\-\-max\-sendto\-failures\fR +Maximum NIC sendto failures before scan is aborted +. +.TP +\fB\-\-min\-hitrate\fR +Minimum hitrate that scan can hit before scan is aborted +. +.TP +\fB\-\-cores\fR +Comma\-separated list of cores to pin to +. +.TP +\fB\-\-ignore\-blocklist\-errors\fR +Ignore invalid, malformed, or unresolvable entries in allowlist/blocklist file\. Replaces the pre\-v3\.x \fB\-\-ignore\-invalid\-hosts\fR option\. +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help and exit +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit +. +.SS "OUTPUT FILTERS" +Results generated by a probe module can be filtered before being passed to the output module\. Filters are defined over the output fields of a probe module\. Filters are written in a simple filtering language, similar to SQL, and are passed to ZMap using the \fB\-\-output\-filter\fR option\. Output filters are commonly used to filter out duplicate results, or to only pass only successful responses to the output module\. +. +.P +Filter expressions are of the form \fB<fieldname> <operation> <value>\fR\. The type of \fB<value>\fR must be either a string or unsigned integer literal, and match the type of \fB<fieldname>\fR\. The valid operations for integer comparisons are = !=, \fI,\fR, \fI=,\fR=\. The operations for string comparisons are =, !=\. The \fB\-\-list\-output\-fields\fR flag will print what fields and types are available for the selected probe module, and then exit\. +. +.P +Compound filter expressions may be constructed by combining filter expressions using parenthesis to specify order of operations, the && (logical AND) and || (logical OR) operators\. +. +.P +For example, a filter for only successful, non\-duplicate responses would be written as: \fB\-\-output\-filter="success = 1 && repeat = 0"\fR +. +.SS "UDP PROBE MODULE OPTIONS" +These arguments are all passed using the \fB\-\-probe\-args=args\fR option\. Only one argument may be passed at a time\. +. +.TP +\fBfile:/path/to/file\fR +Path to payload file to send to each host over UDP\. +. +.TP +\fBtemplate:/path/to/template\fR +Path to template file\. For each destination host, the template file is populated, set as the UDP payload, and sent\. +. +.TP +\fBtext:<text>\fR +ASCII text to send to each destination host +. +.TP +\fBhex:<hex>\fR +Hex\-encoded binary to send to each destination host +. +.TP +\fBtemplate\-fields\fR +Print information about the allowed template fields and exit\. +. +.SS "MID\-SCAN CHANGES" +You can change the rate at which ZMap is scanning mid\-scan by sending SIGUSR1 (increase) and SIGUSR2 (decrease) signals to ZMap\. These will result in the scan rate increasing or decreasing by 5%\. diff --git a/rdns_scan/zmap4rdns/src/zmap.1.ronn b/rdns_scan/zmap4rdns/src/zmap.1.ronn new file mode 100644 index 0000000..fa6651d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zmap.1.ronn @@ -0,0 +1,282 @@ +zmap(1) - The Fast Internet Scanner +=================================== + +## SYNOPSIS + +zmap [ -p <port> ] [ -o <outfile> ] [ OPTIONS... ] [ ip/hostname/range ] + +## DESCRIPTION + +*ZMap* is a network tool for scanning the entire IPv4 address space (or large +samples). ZMap is capable of scanning the entire Internet in around 45 minutes +on a gigabit network connection, reaching ~98% theoretical line speed. + +## OPTIONS + +### BASIC OPTIONS ### + + * `ip`/`hostname`/`range`: + IP addresses or DNS hostnames to scan. Accepts IP ranges in CIDR block + notation. Defaults to 0.0.0/8 + + * `-p`, `--target-port=port`: + TCP or UDP port number to scan (for SYN scans and basic UDP scans) + + * `-o`, `--output-file=name`: + When using an output module that uses a file, write results to this file. + Use - for stdout. + + * `-b`, `--blocklist-file=path`: + File of subnets to exclude, in CIDR notation, one-per line. It is + recommended you use this to exclude RFC 1918 addresses, multicast, IANA + reserved space, and other IANA special-purpose addresses. An example + blocklist file **blocklist.conf** for this purpose. + + * `-w`, `--allowlist-file=path`: + File of subnets to scan, in CIDR notation, one-per line. Specifying a + allowlist file is equivalent to specifying to ranges directly on the command + line interface, but allows specifying a large number of subnets. Note: + if you are specifying a large number of individual IP addresses (more than + 10 million), you should instead use `--list-of-ips-file`. + + * `-I`, `--list-of-ips-file=path`: + File of individual IP addresses to scan, one-per line. This feature allows you + to scan a large number of unrelated addresses. If you have a small number of IPs, + it is faster to specify these on the command + line or by using `--allowlist-file`. This should only be used when scanning more than + 10 million addresses. When used in with --allowlist-path, only hosts in the intersection + of both sets will be scanned. Hosts specified here, but included in the blocklist will + be excluded. + +### SCAN OPTIONS ### + + * `-r`, `--rate=pps`: + Set the send rate in packets/sec. Note: when combined with --probes, this is + total packets per second, not IPs per second. Setting the rate to 0 will scan + at full line rate. Default: 10000 pps. + + * `-B`, `--bandwidth=bps`: + Set the send rate in bits/second (supports suffixes G, M, and K (e.g. -B + 10M for 10 mbps). This overrides the --rate flag. + + * `-n`, `--max-targets=n`: + Cap the number of targets to probe. This can either be a number (e.g. -n + 1000) or a percentage (e.g. -n 0.1%) of the scannable address space + (after excluding blocklist) + + * `-N`, `--max-results=n`: + Exit after receiving this many results + + * `-t`, `--max-runtime=secs`: + Cap the length of time for sending packets + + * `-c`, `--cooldown-time=secs`: + How long to continue receiving after sending has completed (default=8) + + * `-e`, `--seed=n`: + Seed used to select address permutation. Use this if you want to scan + addresses in the same order for multiple ZMap runs. + + * `-P`, `--probes=n`: + Number of probes to send to each IP (default=1) + + * `--retries=n`: + Number of times to try resending a packet if the sendto call fails (default=10) + + * `--batch=n`: + Number of packets to send in a burst between checks to the ratelimit. A + batch size above 1 allows the sleep-based rate-limiter to be used with + proportionally higher rates. This can reduce CPU usage, in exchange for a + bursty send rate. (default=1) + +### SCAN SHARDING ### + + * `--shards=N`: + Split the scan up into N shards/partitions among different instances of + zmap (default=1). When sharding, **--seed** is required. + + * `--shard=n`: + Set which shard to scan (default=0). Shards are 0-indexed in the range + [0, N), where N is the total number of shards. When sharding + **--seed** is required. + +### NETWORK OPTIONS ### + + * `-s`, `--source-port=port|range`: + Source port(s) to send packets from + + * `-S`, `--source-ip=ip|range`: + Source address(es) to send packets from. Either single IP or range (e.g. + 10.0.0.1-10.0.0.9) + + * `-G`, `--gateway-mac=addr`: + Gateway MAC address to send packets to (in case auto-detection fails) + + * `--source-mac=addr`: + Source MAC address to send packets from (in case auto-detection fails) + + * `-i`, `--interface=name`: + Network interface to use + + * `-X`, `--iplayer`: + Send IP layer packets instead of ethernet packets (for non-Ethernet interface) + +### PROBE OPTIONS ### + +ZMap allows users to specify and write their own probe modules. Probe modules +are responsible for generating probe packets to send, and processing responses +from hosts. + + * `--list-probe-modules`: + List available probe modules (e.g. tcp_synscan) + + * `-M`, `--probe-module=name`: + Select probe module (default=tcp_synscan) + + * `--probe-args=args`: + Arguments to pass to probe module + + * `--probe-ttl=hops`: + Set TTL value for probe IP packets + + * `--list-output-fields`: + List the fields the selected probe module can send to the output module + +### OUTPUT OPTIONS ### + +ZMap allows users to specify and write their own output modules for use with +ZMap. Output modules are responsible for processing the fieldsets returned by +the probe module, and outputting them to the user. Users can specify output +fields, and write filters over the output fields. + + * `--list-output-modules`: + List available output modules (e.g. csv) + + * `-O`, `--output-module=name`: + Select output module (default=csv) + + * `--output-args=args`: + Arguments to pass to output module + + * `-f`, `--output-fields=fields`: + Comma-separated list of fields to output + + * `--output-filter`: + Specify an output filter over the fields defined by the probe module. See + the output filter section for more details. + + * `--no-header-row`: + Excludes any header rows (e.g., CSV header fields) from ZMap output. This is + useful if you're piping results into another application that expects only + data. + +### LOGGING AND METADATA OPTIONS ### + + * `-q`, `--quiet`: + Do not print status updates once per second + + * `-v`, `--verbosity=n`: + Level of log detail (0-5, default=3) + + * `-l`, `--log-file=filename`: + Output file for log messages. By default, stderr. + + * `-m`, `--metadata-file=filename`: + Output file for scan metadata (JSON) + + * `-L`, `--log-directory`: + Write log entries to a timestamped file in this directory + + * `-u`, `--status-updates-file`: + Write scan progress updates to CSV file" + + * `--disable-syslog`: + Disables logging messages to syslog + + * `--notes`: + Inject user-specified notes into scan metadata + + * `--user-metadata`: + Inject user-specified JSON metadata into scan metadata + +### ADDITIONAL OPTIONS ### + + * `-T`, `--sender-threads=n`: + Threads used to send packets. ZMap will attempt to detect the optimal + number of send threads based on the number of processor cores. + + * `-C`, `--config=filename`: + Read a configuration file, which can specify any other options. + + * `-d`, `--dryrun`: + Print out each packet to stdout instead of sending it (useful for + debugging) + + * `--max-sendto-failures`: + Maximum NIC sendto failures before scan is aborted + + * `--min-hitrate`: + Minimum hitrate that scan can hit before scan is aborted + + * `--cores`: + Comma-separated list of cores to pin to + + * `--ignore-blocklist-errors`: + Ignore invalid, malformed, or unresolvable entries in allowlist/blocklist file. + Replaces the pre-v3.x `--ignore-invalid-hosts` option. + + * `-h`, `--help`: + Print help and exit + + * `-V`, `--version`: + Print version and exit + +### OUTPUT FILTERS ### + +Results generated by a probe module can be filtered before being passed to the +output module. Filters are defined over the output fields of a probe module. +Filters are written in a simple filtering language, similar to SQL, and are +passed to ZMap using the `--output-filter` option. Output filters are commonly +used to filter out duplicate results, or to only pass only successful responses +to the output module. + +Filter expressions are of the form `<fieldname> <operation> <value>`. The type of +`<value>` must be either a string or unsigned integer literal, and match the type +of `<fieldname>`. The valid operations for integer comparisons are = !=, <, >, +<=, >=. The operations for string comparisons are =, !=. The +`--list-output-fields` flag will print what fields and types are available for +the selected probe module, and then exit. + +Compound filter expressions may be constructed by combining filter expressions +using parenthesis to specify order of operations, the && (logical AND) and || +(logical OR) operators. + +For example, a filter for only successful, non-duplicate responses would be +written as: `--output-filter="success = 1 && repeat = 0"` + +### UDP PROBE MODULE OPTIONS ### + +These arguments are all passed using the `--probe-args=args` option. Only one +argument may be passed at a time. + + * `file:/path/to/file`: + Path to payload file to send to each host over UDP. + + * `template:/path/to/template`: + Path to template file. For each destination host, the template file is + populated, set as the UDP payload, and sent. + + * `text:<text>`: + ASCII text to send to each destination host + + * `hex:<hex>`: + Hex-encoded binary to send to each destination host + + * `template-fields`: + Print information about the allowed template fields and exit. + +### MID-SCAN CHANGES ### + +You can change the rate at which ZMap is scanning mid-scan by sending SIGUSR1 (increase) +and SIGUSR2 (decrease) signals to ZMap. These will result in the scan rate increasing or +decreasing by 5%. diff --git a/rdns_scan/zmap4rdns/src/zmap.c b/rdns_scan/zmap4rdns/src/zmap.c new file mode 100644 index 0000000..9ac730d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zmap.c @@ -0,0 +1,943 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <assert.h> +#include <sched.h> +#include <errno.h> +#include <pwd.h> +#include <time.h> + +#include <pcap/pcap.h> +#include <json.h> + +#include <pthread.h> + +#include "../lib/includes.h" +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" +#include "../lib/pbm.h" + +#include "aesrand.h" +#include "zopt.h" +#include "send.h" +#include "recv.h" +#include "state.h" +#include "monitor.h" +#include "get_gateway.h" +#include "filter.h" +#include "summary.h" +#include "utility.h" + +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" + +#ifdef PFRING +#include <pfring_zc.h> +static int32_t distrib_func(pfring_zc_pkt_buff *pkt, pfring_zc_queue *in_queue, + void *arg) +{ + (void)pkt; + (void)in_queue; + (void)arg; + return 0; +} +#endif + +pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; + +typedef struct send_arg { + uint32_t cpu; + sock_t sock; + shard_t *shard; +} send_arg_t; + +typedef struct recv_arg { + uint32_t cpu; +} recv_arg_t; + +typedef struct mon_start_arg { + uint32_t cpu; + iterator_t *it; + pthread_mutex_t *recv_ready_mutex; +} mon_start_arg_t; + +const char *default_help_text = + "By default, ZMap prints out unique, successful " + "IP addresses (e.g., SYN-ACK from a TCP SYN scan) " + "in ASCII form (e.g., 192.168.1.5) to stdout or the specified output " + "file. Internally this is handled by the \"csv\" output module and is " + "equivalent to running zmap --output-module=csv --output-fields=saddr " + "--output-filter=\"success = 1 && repeat = 0\" --no-header-row."; + +static void *start_send(void *arg) +{ + send_arg_t *s = (send_arg_t *)arg; + log_debug("zmap", "Pinning a send thread to core %u", s->cpu); + set_cpu(s->cpu); + send_run(s->sock, s->shard); + free(s); + return NULL; +} + +static void *start_recv(void *arg) +{ + recv_arg_t *r = (recv_arg_t *)arg; + log_debug("zmap", "Pinning receive thread to core %u", r->cpu); + set_cpu(r->cpu); + recv_run(&recv_ready_mutex); + return NULL; +} + +static void *start_mon(void *arg) +{ + mon_start_arg_t *mon_arg = (mon_start_arg_t *)arg; + log_debug("zmap", "Pinning monitor thread to core %u", mon_arg->cpu); + set_cpu(mon_arg->cpu); + monitor_run(mon_arg->it, mon_arg->recv_ready_mutex); + free(mon_arg); + return NULL; +} + +static void start_zmap(void) +{ + if (zconf.iface == NULL) { + zconf.iface = get_default_iface(); + assert(zconf.iface); + log_debug("zmap", + "no interface provided. will use default" + " interface (%s).", + zconf.iface); + } + if (zconf.number_source_ips == 0) { + struct in_addr default_ip; + if (get_iface_ip(zconf.iface, &default_ip) < 0) { + log_fatal("zmap", + "could not detect default IP address for %s." + " Try specifying a source address (-S).", + zconf.iface); + } + zconf.source_ip_addresses[0] = default_ip.s_addr; + zconf.number_source_ips++; + log_debug( + "zmap", + "no source IP address given. will use default address: %s.", + inet_ntoa(default_ip)); + } + if (!zconf.gw_mac_set) { + struct in_addr gw_ip; + memset(&gw_ip, 0, sizeof(struct in_addr)); + if (get_default_gw(&gw_ip, zconf.iface) < 0) { + log_fatal( + "zmap", + "could not detect default gateway address for %s." + " Try setting default gateway mac address (-G)." + " If this is a newly launched machine, try completing an outgoing network connection (e.g. curl https://zmap.io), and trying again.", + zconf.iface); + } + log_debug("zmap", "found gateway IP %s on %s", inet_ntoa(gw_ip), + zconf.iface); + zconf.gw_ip = gw_ip.s_addr; + memset(&zconf.gw_mac, 0, MAC_ADDR_LEN); + if (get_hw_addr(&gw_ip, zconf.iface, zconf.gw_mac)) { + log_fatal( + "zmap", + "could not detect GW MAC address for %s on %s." + " Try setting default gateway mac address (-G), or run" + " \"arp <gateway_ip>\" in terminal." + " If this is a newly launched machine, try completing an outgoing network connection (e.g. curl https://zmap.io), and trying again.", + inet_ntoa(gw_ip), zconf.iface); + } + zconf.gw_mac_set = 1; + } + log_debug("send", "gateway MAC address %02x:%02x:%02x:%02x:%02x:%02x", + zconf.gw_mac[0], zconf.gw_mac[1], zconf.gw_mac[2], + zconf.gw_mac[3], zconf.gw_mac[4], zconf.gw_mac[5]); + // Initialization + assert(zconf.output_module && "no output module set"); + log_debug("zmap", "output module: %s", zconf.output_module->name); + if (zconf.output_module && zconf.output_module->init) { + if (zconf.output_module->init(&zconf, zconf.output_fields, + zconf.output_fields_len)) { + log_fatal( + "zmap", + "output module did not initialize successfully."); + } + } + + iterator_t *it = send_init(); + if (!it) { + log_fatal("zmap", "unable to initialize sending component"); + } + if (zconf.output_module && zconf.output_module->start) { + zconf.output_module->start(&zconf, &zsend, &zrecv); + } + + // start threads + uint32_t cpu = 0; + pthread_t *tsend, trecv, tmon; + int r; + if (!zconf.dryrun) { + recv_arg_t *recv_arg = xmalloc(sizeof(recv_arg_t)); + recv_arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + cpu += 1; + r = pthread_create(&trecv, NULL, start_recv, recv_arg); + if (r != 0) { + log_fatal("zmap", "unable to create recv thread"); + } + for (;;) { + pthread_mutex_lock(&recv_ready_mutex); + if (zconf.recv_ready) { + pthread_mutex_unlock(&recv_ready_mutex); + break; + } + pthread_mutex_unlock(&recv_ready_mutex); + } + } +#ifdef PFRING + pfring_zc_worker *zw = pfring_zc_run_balancer( + zconf.pf.queues, &zconf.pf.send, zconf.senders, 1, + zconf.pf.prefetches, round_robin_bursts_policy, NULL, distrib_func, + NULL, 0, zconf.pin_cores[cpu & zconf.pin_cores_len]); + cpu += 1; +#endif + tsend = xmalloc(zconf.senders * sizeof(pthread_t)); + for (uint8_t i = 0; i < zconf.senders; i++) { + sock_t sock; + if (zconf.dryrun) { + sock = get_dryrun_socket(); + } else { + sock = get_socket(i); + } + send_arg_t *arg = xmalloc(sizeof(send_arg_t)); + + arg->sock = sock; + arg->shard = get_shard(it, i); + arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + cpu += 1; + int r = pthread_create(&tsend[i], NULL, start_send, arg); + if (r != 0) { + log_fatal("zmap", "unable to create send thread"); + exit(EXIT_FAILURE); + } + } + log_debug("zmap", "%d sender threads spawned", zconf.senders); + + if (!zconf.dryrun) { + monitor_init(); + mon_start_arg_t *mon_arg = xmalloc(sizeof(mon_start_arg_t)); + mon_arg->it = it; + mon_arg->recv_ready_mutex = &recv_ready_mutex; + mon_arg->cpu = zconf.pin_cores[cpu % zconf.pin_cores_len]; + int r = pthread_create(&tmon, NULL, start_mon, mon_arg); + if (r != 0) { + log_fatal("zmap", "unable to create monitor thread"); + exit(EXIT_FAILURE); + } + } + +#ifndef PFRING + drop_privs(); +#endif + + // wait for completion + for (uint8_t i = 0; i < zconf.senders; i++) { + int r = pthread_join(tsend[i], NULL); + if (r != 0) { + log_fatal("zmap", "unable to join send thread"); + exit(EXIT_FAILURE); + } + } + log_debug("zmap", "senders finished"); +#ifdef PFRING + pfring_zc_kill_worker(zw); + pfring_zc_sync_queue(zconf.pf.send, tx_only); + log_debug("zmap", "send queue flushed"); +#endif + // no receiving or monitoring thread is started in dry run mode + if (!zconf.dryrun) { + r = pthread_join(trecv, NULL); + if (r != 0) { + log_fatal("zmap", "unable to join recv thread"); + exit(EXIT_FAILURE); + } + if (!zconf.quiet || zconf.status_updates_file) { + pthread_join(tmon, NULL); + if (r != 0) { + log_fatal("zmap", + "unable to join monitor thread"); + exit(EXIT_FAILURE); + } + } + } + + // finished + if (zconf.metadata_filename) { + json_metadata(zconf.metadata_file); + } + if (zconf.output_module && zconf.output_module->close) { + zconf.output_module->close(&zconf, &zsend, &zrecv); + } + if (zconf.probe_module && zconf.probe_module->close) { + zconf.probe_module->close(&zconf, &zsend, &zrecv); + } +#ifdef PFRING + pfring_zc_destroy_cluster(zconf.pf.cluster); +#endif + log_info("zmap", "completed"); +} + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char *argv[]) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + int config_loaded = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + if (args.config_given || file_exists(args.config_arg)) { + params->initialize = 0; + params->override = 0; + if (cmdline_parser_config_file(args.config_arg, &args, + params) != 0) { + exit(EXIT_FAILURE); + } + config_loaded = 1; + } + + // set defaults before loading in command line arguments + init_empty_global_configuration(&zconf); + // initialize logging. if no log file or log directory are specified + // default to using stderr. + zconf.log_level = args.verbosity_arg; + zconf.log_file = args.log_file_arg; + zconf.log_directory = args.log_directory_arg; + if (args.disable_syslog_given) { + zconf.syslog = 0; + } else { + zconf.syslog = 1; + } + if (zconf.log_file && zconf.log_directory) { + log_init(stderr, zconf.log_level, zconf.syslog, "zmap"); + log_fatal("zmap", "log-file and log-directory cannot " + "specified simultaneously."); + } + FILE *log_location = NULL; + if (zconf.log_file) { + log_location = fopen(zconf.log_file, "w"); + } else if (zconf.log_directory) { + time_t now; + time(&now); + struct tm *local = localtime(&now); + char path[100]; + strftime(path, 100, "zmap-%Y-%m-%dT%H%M%S%z.log", local); + char *fullpath = + xmalloc(strlen(zconf.log_directory) + strlen(path) + 2); + sprintf(fullpath, "%s/%s", zconf.log_directory, path); + log_location = fopen(fullpath, "w"); + free(fullpath); + } else { + log_location = stderr; + } + if (!log_location) { + log_init(stderr, zconf.log_level, zconf.syslog, "zmap"); + log_fatal("zmap", "unable to open specified log file: %s", + strerror(errno)); + } + log_init(log_location, zconf.log_level, zconf.syslog, "zmap"); + log_debug("zmap", "zmap main thread started"); + if (config_loaded) { + log_debug("zmap", "Loaded configuration file %s", + args.config_arg); + } + if (zconf.syslog) { + log_debug("zmap", "syslog support enabled"); + } else { + log_info("zmap", "syslog support disabled"); + } + // parse the provided probe and output module s.t. that we can support + // other command-line helpers (e.g. probe help) + log_debug("zmap", "requested ouput-module: %s", args.output_module_arg); + + // ZMap's default behavior is to provide a simple file of the unique IP + // addresses that responded successfully. We only use this simple "default" + // mode if none of {output module, output filter, output fields} are set. + zconf.default_mode = (!(args.output_module_given || args.output_filter_given || args.output_fields_given)); + if (zconf.default_mode) { + log_info("zmap", "By default, ZMap will output the unique IP addresses " + "of hosts that respond successfully (e.g., SYN-ACK packet). This " + "is equivalent to running ZMap with the following flags: " + "--output-module=csv --output-fields=saddr --output-filter='" + "success=1 && repeat=0' --no-header-row. " + "If you want all responses, explicitly set an output module or " + "set --output-filter=\"\"."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.output_module_name = strdup("csv"); + zconf.no_header_row = 1; + } else if (!args.output_module_given) { + log_debug("zmap", "No output module provided. Will use csv."); + zconf.output_module = get_output_module_by_name("csv"); + zconf.output_module_name = strdup("csv"); + } else { + zconf.output_module = + get_output_module_by_name(args.output_module_arg); + if (!zconf.output_module) { + log_fatal( + "zmap", + "specified output module (%s) does not exist\n", + args.output_module_arg); + } + zconf.output_module_name = strdup(args.output_module_arg); + } + zconf.probe_module = get_probe_module_by_name(args.probe_module_arg); + if (!zconf.probe_module) { + log_fatal("zmap", + "specified probe module (%s) does not exist\n", + args.probe_module_arg); + exit(EXIT_FAILURE); + } + // check whether the probe module is going to generate dynamic data + // and that the output module can support exporting that data out of + // zmap. If they can't, then quit. + if (zconf.probe_module->output_type == OUTPUT_TYPE_DYNAMIC && + !zconf.output_module->supports_dynamic_output) { + log_fatal( + "zmap", + "specified probe module (%s) requires dynamic " + "output support, which output module (%s) does not support. " + "Most likely you want to use JSON output.", + args.probe_module_arg, args.output_module_arg); + } + if (args.help_given) { + cmdline_parser_print_help(); + printf("\nProbe Module (%s) Help:\n", zconf.probe_module->name); + if (zconf.probe_module->helptext) { + fprintw(stdout, zconf.probe_module->helptext, + 80); + } else { + printf("no help text available\n"); + } + assert(zconf.output_module && "no output module set"); + const char *module_name = zconf.default_mode ? "Default" : zconf.output_module->name; + printf("\nOutput Module (%s) Help:\n", module_name); + + if (zconf.default_mode) { + fprintw(stdout, default_help_text, 80); + } else if (zconf.output_module->helptext) { + fprintw(stdout, zconf.output_module->helptext, + 80); + } else { + printf("no help text available\n"); + } + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + if (args.list_output_modules_given) { + print_output_modules(); + exit(EXIT_SUCCESS); + } + if (args.list_probe_modules_given) { + print_probe_modules(); + exit(EXIT_SUCCESS); + } + if (args.iplayer_given) { + zconf.send_ip_pkts = 1; + zconf.gw_mac_set = 1; + memset(zconf.gw_mac, 0, MAC_ADDR_LEN); + } + if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) { + exit(EXIT_FAILURE); + } + // now that we know the probe module, let's find what it supports + memset(&zconf.fsconf, 0, sizeof(struct fieldset_conf)); + // the set of fields made available to a user is constructed + // of IP header fields + probe module fields + system fields + fielddefset_t *fds = &(zconf.fsconf.defs); + gen_fielddef_set(fds, (fielddef_t *)&(ip_fields), ip_fields_len); + gen_fielddef_set(fds, zconf.probe_module->fields, + zconf.probe_module->numfields); + gen_fielddef_set(fds, (fielddef_t *)&(sys_fields), sys_fields_len); + if (args.list_output_fields_given) { + for (int i = 0; i < fds->len; i++) { + printf("%-15s %6s: %s\n", fds->fielddefs[i].name, + fds->fielddefs[i].type, fds->fielddefs[i].desc); + } + exit(EXIT_SUCCESS); + } + // find the fields we need for the framework + zconf.fsconf.success_index = + fds_get_index_by_name(fds, "success"); + if (zconf.fsconf.success_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required success field."); + } + zconf.fsconf.app_success_index = fds_get_index_by_name(fds, "app_success"); + + if (zconf.fsconf.app_success_index < 0) { + log_debug("fieldset", "probe module does not supply " + "application success field."); + } else { + log_debug( "fieldset", + "probe module supplies app_success" + " output field. It will be included in monitor output"); + } + zconf.fsconf.classification_index = fds_get_index_by_name(fds, "classification"); + if (zconf.fsconf.classification_index < 0) { + log_fatal("fieldset", "probe module does not supply " + "required packet classification field."); + } + // process the list of requested output fields. + if (args.output_fields_given) { + zconf.raw_output_fields = args.output_fields_arg; + } else { + zconf.raw_output_fields = "saddr"; + } + // add all fields if wildcard received + if (!strcmp(zconf.raw_output_fields, "*")) { + zconf.output_fields_len = zconf.fsconf.defs.len; + zconf.output_fields = + xcalloc(zconf.fsconf.defs.len, sizeof(const char *)); + for (int i = 0; i < zconf.fsconf.defs.len; i++) { + zconf.output_fields[i] = + zconf.fsconf.defs.fielddefs[i].name; + } + fs_generate_full_fieldset_translation(&zconf.fsconf.translation, + &zconf.fsconf.defs); + } else { + split_string(zconf.raw_output_fields, + &(zconf.output_fields_len), + &(zconf.output_fields)); + for (int i = 0; i < zconf.output_fields_len; i++) { + log_debug("zmap", "requested output field (%i): %s", i, + zconf.output_fields[i]); + } + // generate a translation that can be used to convert output + // from a probe module to the input for an output module + fs_generate_fieldset_translation( + &zconf.fsconf.translation, &zconf.fsconf.defs, + zconf.output_fields, zconf.output_fields_len); + } + // default filtering behavior is to drop unsuccessful and duplicates + if (zconf.default_mode) { + log_debug( + "filter", + "No output filter specified. Will use default: exclude duplicates and unssuccessful"); + } else if (args.output_filter_given && strcmp(args.output_filter_arg, "")) { + // Run it through yyparse to build the expression tree + if (!parse_filter_string(args.output_filter_arg)) { + log_fatal("zmap", "Unable to parse filter expression"); + } + // Check the fields used against the fieldset in use + if (!validate_filter(zconf.filter.expression, + &zconf.fsconf.defs)) { + log_fatal("zmap", "Invalid filter"); + } + zconf.output_filter_str = args.output_filter_arg; + log_debug("filter", "will use output filter %s", + args.output_filter_arg); + } else if (args.output_filter_given) { // (empty filter argument) + log_debug("filter", "Empty output filter provided. ZMap will output all " + "results, including duplicate and non-successful responses."); + } else { + log_info("filter", "No output filter provided. ZMap will output all " + "results, including duplicate and non-successful responses (e.g., " + "RST and ICMP packets). If you want a filter similar to ZMap's " + "default behavior, you can set an output filter similar to the " + "following: --output-filter=\"success=1 && repeat=0\"."); + } + zconf.ignore_invalid_hosts = args.ignore_blocklist_errors_given; + + SET_BOOL(zconf.dryrun, dryrun); + SET_BOOL(zconf.quiet, quiet); + SET_BOOL(zconf.no_header_row, no_header_row); + zconf.cooldown_secs = args.cooldown_time_arg; + SET_IF_GIVEN(zconf.output_filename, output_file); + SET_IF_GIVEN(zconf.blocklist_filename, blocklist_file); + SET_IF_GIVEN(zconf.list_of_ips_filename, list_of_ips_file); + SET_IF_GIVEN(zconf.probe_args, probe_args); + SET_IF_GIVEN(zconf.probe_ttl, probe_ttl); + SET_IF_GIVEN(zconf.output_args, output_args); + SET_IF_GIVEN(zconf.iface, interface); + SET_IF_GIVEN(zconf.max_runtime, max_runtime); + SET_IF_GIVEN(zconf.max_results, max_results); + SET_IF_GIVEN(zconf.rate, rate); + SET_IF_GIVEN(zconf.packet_streams, probes); + SET_IF_GIVEN(zconf.status_updates_file, status_updates_file); + SET_IF_GIVEN(zconf.num_retries, retries); + SET_IF_GIVEN(zconf.max_sendto_failures, max_sendto_failures); + SET_IF_GIVEN(zconf.min_hitrate, min_hitrate); + + if (zconf.num_retries < 0) { + log_fatal("zmap", "Invalid retry count"); + } + + if (zconf.max_sendto_failures >= 0) { + log_debug("zmap", + "scan will abort if more than %i " + "sendto failures occur", + zconf.max_sendto_failures); + } + if (zconf.min_hitrate > 0.0) { + log_debug("zmap", "scan will abort if hitrate falls below %f", + zconf.min_hitrate); + } + if (args.metadata_file_arg) { + zconf.metadata_filename = args.metadata_file_arg; + if (!strcmp(zconf.metadata_filename, "-")) { + zconf.metadata_file = stdout; + } else { + zconf.metadata_file = + fopen(zconf.metadata_filename, "w"); + } + if (!zconf.metadata_file) { + log_fatal("metadata", + "unable to open metadata file (%s): %s", + zconf.metadata_filename, strerror(errno)); + } + log_debug("metadata", "metadata will be saved to %s", + zconf.metadata_filename); + } + + if (args.user_metadata_given) { + zconf.custom_metadata_str = args.user_metadata_arg; + if (!json_tokener_parse(zconf.custom_metadata_str)) { + log_fatal("metadata", + "unable to parse custom user metadata"); + } else { + log_debug("metadata", + "user metadata validated successfully"); + } + } + if (args.notes_given) { + zconf.notes = args.notes_arg; + } + + // find if zmap wants any specific cidrs scanned instead + // of the entire Internet + zconf.destination_cidrs = args.inputs; + zconf.destination_cidrs_len = args.inputs_num; + if (zconf.destination_cidrs && zconf.blocklist_filename && + !strcmp(zconf.blocklist_filename, "/etc/zmap/blocklist.conf")) { + log_warn( + "blocklist", + "ZMap is currently using the default blocklist located " + "at /etc/zmap/blocklist.conf. By default, this blocklist excludes locally " + "scoped networks (e.g. 10.0.0.0/8, 127.0.0.1/8, and 192.168.0.0/16). If you are" + " trying to scan local networks, you can change the default blocklist by " + "editing the default ZMap configuration at /etc/zmap/zmap.conf."); + } + SET_IF_GIVEN(zconf.allowlist_filename, allowlist_file); + + if (zconf.probe_module->port_args) { + if (args.source_port_given) { + char *dash = strchr(args.source_port_arg, '-'); + if (dash) { // range + *dash = '\0'; + zconf.source_port_first = + atoi(args.source_port_arg); + enforce_range("starting source-port", + zconf.source_port_first, 0, + 0xFFFF); + zconf.source_port_last = atoi(dash + 1); + enforce_range("ending source-port", + zconf.source_port_last, 0, + 0xFFFF); + if (zconf.source_port_first > + zconf.source_port_last) { + fprintf( + stderr, + "%s: invalid source port range: " + "last port is less than first port\n", + CMDLINE_PARSER_PACKAGE); + exit(EXIT_FAILURE); + } + } else { // single port + int port = atoi(args.source_port_arg); + enforce_range("source-port", port, 0, 0xFFFF); + zconf.source_port_first = port; + zconf.source_port_last = port; + } + } + if (!args.target_port_given) { + log_fatal( + "zmap", + "target port (-p) is required for this type of probe"); + } + enforce_range("target-port", args.target_port_arg, 0, 0xFFFF); + zconf.target_port = args.target_port_arg; + } + if (args.source_ip_given) { + parse_source_ip_addresses(args.source_ip_arg); + } + if (args.gateway_mac_given) { + if (!parse_mac(zconf.gw_mac, args.gateway_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + zconf.gw_mac_set = 1; + } + if (args.source_mac_given) { + if (!parse_mac(zconf.hw_mac, args.source_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + log_debug("send", + "source MAC address specified on CLI: " + "%02x:%02x:%02x:%02x:%02x:%02x", + zconf.hw_mac[0], zconf.hw_mac[1], zconf.hw_mac[2], + zconf.hw_mac[3], zconf.hw_mac[4], zconf.hw_mac[5]); + + zconf.hw_mac_set = 1; + } + // Check for a random seed + if (args.seed_given) { + zconf.seed = args.seed_arg; + zconf.seed_provided = 1; + } else { + // generate a seed randomly + if (!random_bytes(&zconf.seed, sizeof(uint64_t))) { + log_fatal("zmap", "unable to generate random bytes " + "needed for seed"); + } + zconf.seed_provided = 0; + } + zconf.aes = aesrand_init_from_seed(zconf.seed); + + // Set up sharding + zconf.shard_num = 0; + zconf.total_shards = 1; + if ((args.shard_given || args.shards_given) && !args.seed_given) { + log_fatal("zmap", "Need to specify seed if sharding a scan"); + } + if (args.shard_given ^ args.shards_given) { + log_fatal( + "zmap", + "Need to specify both shard number and total number of shards"); + } + if (args.shard_given) { + enforce_range("shard", args.shard_arg, 0, 65534); + } + if (args.shards_given) { + enforce_range("shards", args.shards_arg, 1, 65535); + } + SET_IF_GIVEN(zconf.shard_num, shard); + SET_IF_GIVEN(zconf.total_shards, shards); + if (zconf.shard_num >= zconf.total_shards) { + log_fatal("zmap", + "With %hhu total shards, shard number (%hhu)" + " must be in range [0, %hhu)", + zconf.total_shards, zconf.shard_num, + zconf.total_shards); + } + + if (args.bandwidth_given) { + // Supported: G,g=*1000000000; M,m=*1000000 K,k=*1000 bits per + // second + zconf.bandwidth = atoi(args.bandwidth_arg); + char *suffix = args.bandwidth_arg; + while (*suffix >= '0' && *suffix <= '9') { + suffix++; + } + if (*suffix) { + switch (*suffix) { + case 'G': + case 'g': + zconf.bandwidth *= 1000000000; + break; + case 'M': + case 'm': + zconf.bandwidth *= 1000000; + break; + case 'K': + case 'k': + zconf.bandwidth *= 1000; + break; + default: + fprintf(stderr, + "%s: unknown bandwidth suffix '%s' " + "(supported suffixes are G, M and K)\n", + CMDLINE_PARSER_PACKAGE, suffix); + exit(EXIT_FAILURE); + } + } + } + if(args.batch_given){ + zconf.batch = args.batch_arg; + } + if (args.max_targets_given) { + zconf.max_targets = parse_max_hosts(args.max_targets_arg); + } + + // blocklist + if (blocklist_init(zconf.allowlist_filename, zconf.blocklist_filename, + zconf.destination_cidrs, zconf.destination_cidrs_len, + NULL, 0, zconf.ignore_invalid_hosts)) { + log_fatal("zmap", "unable to initialize blocklist / allowlist"); + } + // if there's a list of ips to scan, then initialize PBM and populate + // it based on the provided file + if (zconf.list_of_ips_filename) { + zsend.list_of_ips_pbm = pbm_init(); + zconf.list_of_ips_count = pbm_load_from_file( + zsend.list_of_ips_pbm, zconf.list_of_ips_filename); + } + + // compute number of targets + uint64_t allowed = blocklist_count_allowed(); + zconf.total_allowed = allowed; + zconf.total_disallowed = blocklist_count_not_allowed(); + assert(allowed <= (1LL << 32)); + if (!zconf.total_allowed) { + log_fatal("zmap", "zero eligible addresses to scan"); + } + if (zconf.list_of_ips_count > 0 && 0xFFFFFFFFU / zconf.list_of_ips_count > 100000) { + log_warn("zmap", "list of IPs is small compared to address space. Performance will suffer, consider using an allowlist instead"); + } + if (zconf.max_targets) { + zsend.max_targets = zconf.max_targets; + } +#ifndef PFRING + // Set the correct number of threads, default to num_cores - 1 + if (args.sender_threads_given) { + zconf.senders = args.sender_threads_arg; + } else { + zconf.senders = 1; + } + if (2 * zconf.senders >= zsend.max_targets) { + log_warn( + "zmap", + "too few targets relative to senders, dropping to one sender"); + zconf.senders = 1; + } +#else + zconf.senders = args.sender_threads_arg; +#endif + // Figure out what cores to bind to + if (args.cores_given) { + const char **core_list = NULL; + int len = 0; + split_string(args.cores_arg, &len, &core_list); + zconf.pin_cores_len = (uint32_t)len; + zconf.pin_cores = + xcalloc(zconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < zconf.pin_cores_len; ++i) { + zconf.pin_cores[i] = atoi(core_list[i]); + } + } else { + int num_cores = sysconf(_SC_NPROCESSORS_ONLN); + zconf.pin_cores_len = (uint32_t)num_cores; + zconf.pin_cores = + xcalloc(zconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < zconf.pin_cores_len; ++i) { + zconf.pin_cores[i] = i; + } + } + +// PFRING +#ifdef PFRING +#define MAX_CARD_SLOTS 32768 +#define QUEUE_LEN 8192 +#define ZMAP_PF_BUFFER_SIZE 1536 +#define ZMAP_PF_ZC_CLUSTER_ID 9627 + uint32_t user_buffers = zconf.senders * 256; + uint32_t queue_buffers = zconf.senders * QUEUE_LEN; + uint32_t card_buffers = 2 * MAX_CARD_SLOTS; + uint32_t total_buffers = + user_buffers + queue_buffers + card_buffers + 2; + uint32_t metadata_len = 0; + uint32_t numa_node = 0; // TODO + zconf.pf.cluster = pfring_zc_create_cluster( + ZMAP_PF_ZC_CLUSTER_ID, ZMAP_PF_BUFFER_SIZE, metadata_len, + total_buffers, numa_node, NULL, NULL); + if (zconf.pf.cluster == NULL) { + log_fatal("zmap", "Could not create zc cluster: %s", + strerror(errno)); + } + + zconf.pf.buffers = xcalloc(user_buffers, sizeof(pfring_zc_pkt_buff *)); + for (uint32_t i = 0; i < user_buffers; ++i) { + zconf.pf.buffers[i] = + pfring_zc_get_packet_handle(zconf.pf.cluster); + if (zconf.pf.buffers[i] == NULL) { + log_fatal("zmap", "Could not get ZC packet handle"); + } + } + + zconf.pf.send = + pfring_zc_open_device(zconf.pf.cluster, zconf.iface, tx_only, 0); + if (zconf.pf.send == NULL) { + log_fatal("zmap", "Could not open device %s for TX. [%s]", + zconf.iface, strerror(errno)); + } + + zconf.pf.recv = + pfring_zc_open_device(zconf.pf.cluster, zconf.iface, rx_only, 0); + if (zconf.pf.recv == NULL) { + log_fatal("zmap", "Could not open device %s for RX. [%s]", + zconf.iface, strerror(errno)); + } + + zconf.pf.queues = xcalloc(zconf.senders, sizeof(pfring_zc_queue *)); + for (uint32_t i = 0; i < zconf.senders; ++i) { + zconf.pf.queues[i] = + pfring_zc_create_queue(zconf.pf.cluster, QUEUE_LEN); + if (zconf.pf.queues[i] == NULL) { + log_fatal("zmap", "Could not create queue: %s", + strerror(errno)); + } + } + + zconf.pf.prefetches = pfring_zc_create_buffer_pool(zconf.pf.cluster, 8); + if (zconf.pf.prefetches == NULL) { + log_fatal("zmap", "Could not open prefetch pool: %s", + strerror(errno)); + } +#endif + + // resume scan if requested + + start_zmap(); + + fclose(log_location); + + cmdline_parser_free(&args); + free(params); + return EXIT_SUCCESS; +} diff --git a/rdns_scan/zmap4rdns/src/zmap_schema.py b/rdns_scan/zmap4rdns/src/zmap_schema.py new file mode 100644 index 0000000..05e8423 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zmap_schema.py @@ -0,0 +1,93 @@ +from zschema.leaves import * +from zschema.compounds import * +import zschema.registry + +zmap_base = Record({ + "saddr":IPv4Address(), + "saddr_raw":Unsigned32BitInteger(), + "daddr":IPv4Address(), + "daddr_raw":Unsigned32BitInteger(), + "ipid":Unsigned32BitInteger(), + "ttl":Unsigned32BitInteger(), + "classification":String(), + "success":Unsigned32BitInteger(), + "app_success":Unsigned32BitInteger(), + "repeat":Unsigned32BitInteger(), + "cooldown":Unsigned32BitInteger(), + "timestamp_str":String(), + "timestamp_ts":Unsigned32BitInteger(), + "timestamp_us":Unsigned32BitInteger(), + "icmp_responder":String(), + "icmp_type":Unsigned32BitInteger(), + "icmp_code":Unsigned32BitInteger(), + "icmp_unreach_str":String(), + "sport":Unsigned32BitInteger(), + "dport":Unsigned32BitInteger(), + "data":String(), + "length":Unsigned32BitInteger(), + +}) + +zmap_upnp = Record({ + "type":String(), + "server":AnalyzedString(), + "location":AnalyzedString(), + "usn":String(), + "st":String(), + "ext":String(), + "cache_control":String(), + "x_user_agent":String(), + "agent":String(), + "date":String(), +}, extends=zmap_base) + +zschema.registry.register_schema("zmap-upnp", zmap_upnp) + + +dns_question = SubRecord({ + "name":String(), + "qtype":Unsigned32BitInteger(), + "qtype_str":String(), + "qclass":Unsigned32BitInteger(), +}) + +dns_answer = SubRecord({ + "name":String(), + "type":Unsigned32BitInteger(), + "type_str":String(), + "class":Unsigned32BitInteger(), + "ttl":Unsigned32BitInteger(), + "rdlength":Unsigned32BitInteger(), + "rdata_is_parsed":Unsigned32BitInteger(), + "rdata":String(), # hex +}) + +zmap_dns = Record({ + "qr":Unsigned16BitInteger(), + "rcode":Unsigned16BitInteger(), + "dns_id":Unsigned32BitInteger(), + "dns_rd":Unsigned32BitInteger(), + "dns_tc":Unsigned32BitInteger(), + "dns_aa":Unsigned32BitInteger(), + "dns_opcode":Unsigned32BitInteger(), + "dns_qr":Unsigned32BitInteger(), + "dns_rcode":Unsigned32BitInteger(), + "dns_cd":Unsigned32BitInteger(), + "dns_ad":Unsigned32BitInteger(), + "dns_z":Unsigned32BitInteger(), + "dns_ra":Unsigned32BitInteger(), + "dns_qdcount":Unsigned32BitInteger(), + "dns_ancount":Unsigned32BitInteger(), + "dns_nscount":Unsigned32BitInteger(), + "dns_arcount":Unsigned32BitInteger(), + "dns_questions":ListOf(dns_question), + "dns_answers":ListOf(dns_answer), + "dns_authorities":ListOf(dns_answer), + "dns_additionals":ListOf(dns_answer), + "dns_unconsumed_bytes":Unsigned32BitInteger(), + "dns_parse_err":Unsigned32BitInteger(), + "raw_data":String(), + "udp_len":Unsigned32BitInteger(), +}, extends=zmap_base) + +zschema.registry.register_schema("zmap-dns", zmap_dns) diff --git a/rdns_scan/zmap4rdns/src/zopt.ggo.in b/rdns_scan/zmap4rdns/src/zopt.ggo.in new file mode 100644 index 0000000..9d5d277 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zopt.ggo.in @@ -0,0 +1,203 @@ +# ZMap Copyright 2013 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# zmap option description to be processed by gengetopt + +package "zmap" +version "@ZMAP_VERSION@" +purpose "A fast Internet-wide scanner." + +section "Basic Arguments" + +option "target-port" p "port number to scan (for TCP and UDP scans)" + typestr="port" + optional int +option "output-file" o "Output file" + typestr="name" + optional string +option "blocklist-file" b "File of subnets to exclude, in CIDR notation, e.g. 192.168.0.0/16" + typestr="path" + optional string +option "allowlist-file" w "File of subnets to constrain scan to, in CIDR notation, e.g. 192.168.0.0/16" + typestr="path" + optional string +option "list-of-ips-file" I "List of individual addresses to scan in random order. Use --white-list file unless >1 million IPs" + typestr="path" + optional string + + +section "Scan Options" + +option "rate" r "Set send rate in packets/sec" + typestr="pps" + optional int +option "bandwidth" B "Set send rate in bits/second (supports suffixes G, M and K)" + typestr="bps" + optional string +option "batch" - "Set the number of packets to send per iteration" + typestr="pps" + optional int +option "max-targets" n "Cap number of targets to probe (as a number or a percentage of the address space)" + typestr="n" + optional string +option "max-runtime" t "Cap length of time for sending packets" + typestr="secs" + optional int +option "max-results" N "Cap number of results to return" + typestr="n" + optional int +option "probes" P "Number of probes to send to each IP" + typestr="n" + default="1" + optional int +option "cooldown-time" c "How long to continue receiving after sending last probe" + typestr="secs" + default="8" + optional int +option "seed" e "Seed used to select address permutation" + typestr="n" + optional longlong +option "retries" - "Max number of times to try to send packet if send fails" + typestr="n" + default="10" + optional int +option "dryrun" d "Don't actually send packets" + optional + + +section "Scan Sharding" + +option "shards" - "Set the total number of shards" + typestr="N" + optional int + default="1" +option "shard" - "Set which shard this scan is (0 indexed)" + typestr="n" + optional int + default="0" + +section "Network Options" + +option "source-port" s "Source port(s) for scan packets" + typestr="port|range" + optional string +option "source-ip" S "Source address(es) for scan packets" + typestr="ip|range" + optional string +option "gateway-mac" G "Specify gateway MAC address" + typestr="addr" + optional string +option "source-mac" - "Source MAC address" + typestr="addr" + optional string +option "interface" i "Specify network interface to use" + typestr="name" + optional string +option "iplayer" X "Sends IP packets instead of Ethernet (for VPNs)" + optional + +section "Probe Modules" +option "probe-module" M "Select probe module" + typestr="name" + default="tcp_synscan" + optional string + +option "probe-args" - "Arguments to pass to probe module" + typestr="args" + optional string + +option "probe-ttl" - "Set TTL value for probe IP packets" + typestr="n" + default="255" + optional int + +option "list-probe-modules" - "List available probe modules" + optional + +section "Results Output" +option "output-fields" f "Fields that should be output in result set" + typestr="fields" + optional string +option "output-module" O "Select output module" + typestr="name" + optional string +option "output-args" - "Arguments to pass to output module" + typestr="args" + optional string +option "output-filter" - "Specify a filter over the response fields to limit what responses get sent to the output module" + typestr="filter" + optional string +option "list-output-modules" - "List available output modules" + optional +option "list-output-fields" - "List all fields that can be output by selected probe module" + optional +option "no-header-row" - "Precludes outputting any header rows in data (e.g., CSV headers)" + optional + + +section "Logging and Metadata" +option "verbosity" v "Level of log detail (0-5)" + typestr="n" + default="3" + optional int +option "log-file" l "Write log entries to file" + typestr="name" + optional string +option "log-directory" L "Write log entries to a timestamped file in this directory" + typestr="directory" + optional string +option "metadata-file" m "Output file for scan metadata (JSON)" + typestr="name" + optional string +option "status-updates-file" u "Write scan progress updates to CSV file" + typestr="name" + optional string +option "quiet" q "Do not print status updates" + optional +option "disable-syslog" - "Disables logging messages to syslog" + optional +option "notes" - "Inject user-specified notes into scan metadata" + typestr="notes" + optional string +option "user-metadata" - "Inject user-specified JSON metadata into scan metadata" + typestr="json" + optional string + +section "Additional Options" +option "config" C "Read a configuration file, which can specify any of these options" + typestr="filename" + default="/etc/zmap/zmap.conf" + optional string + +option "max-sendto-failures" - "Maximum NIC sendto failures before scan is aborted" + typestr="n" + default="-1" + optional int + +option "min-hitrate" - "Minimum hitrate that scan can hit before scan is aborted" + typestr="n" + default="0.0" + optional float + +option "sender-threads" T "Threads used to send packets" + typestr="n" + default="1" + optional int + +option "cores" - "Comma-separated list of cores to pin to" + optional string +option "ignore-blocklist-errors" - "Ignore invalid entries in allowlist/blocklist file." + optional +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples:\n\ + zmap -p 80 (scan full IPv4 address space for hosts on TCP/80)\n\ + zmap -N 5 -B 10M -p 80 (find 5 HTTP servers, scanning at 10 Mb/s)\n\ + zmap -p 80 10.0.0.0/8 192.168.0.0/16 (scan both subnets on TCP/80)\n\ + zmap -p 80 1.2.3.4 10.0.0.3 (scan 1.2.3.4, 10.0.0.3 on TCP/80)" diff --git a/rdns_scan/zmap4rdns/src/zopt_compat.c b/rdns_scan/zmap4rdns/src/zopt_compat.c new file mode 100644 index 0000000..f5f406d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/zopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "zopt.c" diff --git a/rdns_scan/zmap4rdns/src/ztee.1 b/rdns_scan/zmap4rdns/src/ztee.1 new file mode 100644 index 0000000..2371761 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztee.1 @@ -0,0 +1,57 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "ZTEE" "1" "September 2017" "ZMap" "ztee" +. +.SH "NAME" +\fBztee\fR \- output buffer and splitter +. +.SH "SYNOPSIS" +ztee [ OPTIONS\.\.\. ] [ FILE\.\.\. ] +. +.SH "DESCRIPTION" +\fIZTee\fR is an output buffer and splitter for use with ZMap output data\. ZTee should be used whenever ZMap is piped into an application scanner, placed between ZMap and the application scanner\. ZTee writes the transformed output to stdout, and writes the original output to FILE\. +. +.P +See \fB\-\-help\fR for examples\. +. +.SH "CSV PROCESSING AND RAW MODE" +\fIZTee\fR operates by default on CSV\-format output from ZMap\. It only outputs IP addresses (from the input\'s \fBip\fR or \fBsaddr\fR field) to stdout, while writing all input to the output file\. ZTee does not print the first line of input to stdout, since that row is the CSV header\. +. +.P +To operate on data in any other format, pass the \fB\-\-raw\fR flag\. In raw mode, ztee behaves like tee: it will not transform or attempt to parse the input data\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-r\fR, \fB\-\-raw\fR +Ignore input formatting and pass through raw input\. This causes ztee to behave exactly like tee, with the addition of buffering\. +. +.TP +\fB\-\-success\-only\fR +Only write to stdout rows where success=1 or success=true\. Invalid in combination with \fB\-\-raw\fR\. +. +.TP +\fB\-m\fR, \fB\-\-monitor\fR +Print monitor data to stderr +. +.TP +\fB\-u\fR, \fB\-\-status\-updates\-file\fR +Write status updates (monitor data) to the given file, in CSV format +. +.TP +\fB\-l\fR, \fB\-\-log\-file=name\fR +Write errors etc\. to the given file\. If none, ZTee logs to stderr\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-h, \-\-help\fR +Display help +. +.TP +\fB\-V, \-\-version\fR +Display version + diff --git a/rdns_scan/zmap4rdns/src/ztee.1.html b/rdns_scan/zmap4rdns/src/ztee.1.html new file mode 100644 index 0000000..81a5397 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztee.1.html @@ -0,0 +1,129 @@ +<!DOCTYPE html> +<html> +<head> + <meta http-equiv='content-type' value='text/html;charset=utf8'> + <meta name='generator' value='Ronn/v0.7.3 (http://github.com/rtomayko/ronn/tree/0.7.3)'> + <title>ztee(1) - output buffer and splitter</title> + <style type='text/css' media='all'> + /* style: man */ + body#manpage {margin:0} + .mp {max-width:100ex;padding:0 9ex 1ex 4ex} + .mp p,.mp pre,.mp ul,.mp ol,.mp dl {margin:0 0 20px 0} + .mp h2 {margin:10px 0 0 0} + .mp > p,.mp > pre,.mp > ul,.mp > ol,.mp > dl {margin-left:8ex} + .mp h3 {margin:0 0 0 4ex} + .mp dt {margin:0;clear:left} + .mp dt.flush {float:left;width:8ex} + .mp dd {margin:0 0 0 9ex} + .mp h1,.mp h2,.mp h3,.mp h4 {clear:left} + .mp pre {margin-bottom:20px} + .mp pre+h2,.mp pre+h3 {margin-top:22px} + .mp h2+pre,.mp h3+pre {margin-top:5px} + .mp img {display:block;margin:auto} + .mp h1.man-title {display:none} + .mp,.mp code,.mp pre,.mp tt,.mp kbd,.mp samp,.mp h3,.mp h4 {font-family:monospace;font-size:14px;line-height:1.42857142857143} + .mp h2 {font-size:16px;line-height:1.25} + .mp h1 {font-size:20px;line-height:2} + .mp {text-align:justify;background:#fff} + .mp,.mp code,.mp pre,.mp pre code,.mp tt,.mp kbd,.mp samp {color:#131211} + .mp h1,.mp h2,.mp h3,.mp h4 {color:#030201} + .mp u {text-decoration:underline} + .mp code,.mp strong,.mp b {font-weight:bold;color:#131211} + .mp em,.mp var {font-style:italic;color:#232221;text-decoration:none} + .mp a,.mp a:link,.mp a:hover,.mp a code,.mp a pre,.mp a tt,.mp a kbd,.mp a samp {color:#0000ff} + .mp b.man-ref {font-weight:normal;color:#434241} + .mp pre {padding:0 4ex} + .mp pre code {font-weight:normal;color:#434241} + .mp h2+pre,h3+pre {padding-left:0} + ol.man-decor,ol.man-decor li {margin:3px 0 10px 0;padding:0;float:left;width:33%;list-style-type:none;text-transform:uppercase;color:#999;letter-spacing:1px} + ol.man-decor {width:100%} + ol.man-decor li.tl {text-align:left} + ol.man-decor li.tc {text-align:center;letter-spacing:4px} + ol.man-decor li.tr {text-align:right;float:right} + </style> +</head> +<!-- + The following styles are deprecated and will be removed at some point: + div#man, div#man ol.man, div#man ol.head, div#man ol.man. + + The .man-page, .man-decor, .man-head, .man-foot, .man-title, and + .man-navigation should be used instead. +--> +<body id='manpage'> + <div class='mp' id='man'> + + <div class='man-navigation' style='display:none'> + <a href="#NAME">NAME</a> + <a href="#SYNOPSIS">SYNOPSIS</a> + <a href="#DESCRIPTION">DESCRIPTION</a> + <a href="#CSV-PROCESSING-AND-RAW-MODE">CSV PROCESSING AND RAW MODE</a> + <a href="#OPTIONS">OPTIONS</a> + </div> + + <ol class='man-decor man-head man head'> + <li class='tl'>ztee(1)</li> + <li class='tc'>ztee</li> + <li class='tr'>ztee(1)</li> + </ol> + + <h2 id="NAME">NAME</h2> +<p class="man-name"> + <code>ztee</code> - <span class="man-whatis">output buffer and splitter</span> +</p> + +<h2 id="SYNOPSIS">SYNOPSIS</h2> + +<p>ztee [ OPTIONS... ] [ FILE... ]</p> + +<h2 id="DESCRIPTION">DESCRIPTION</h2> + +<p><em>ZTee</em> is an output buffer and splitter for use with ZMap output data. ZTee +should be used whenever ZMap is piped into an application scanner, placed +between ZMap and the application scanner. ZTee writes the transformed output +to stdout, and writes the original output to FILE.</p> + +<p>See <code>--help</code> for examples.</p> + +<h2 id="CSV-PROCESSING-AND-RAW-MODE">CSV PROCESSING AND RAW MODE</h2> + +<p><em>ZTee</em> operates by default on CSV-format output from ZMap. It only outputs IP +addresses (from the input's <code>ip</code> or <code>saddr</code> field) to stdout, while writing all +input to the output file. ZTee does not print the first line of input to stdout, +since that row is the CSV header.</p> + +<p>To operate on data in any other format, pass the <code>--raw</code> flag. In raw mode, ztee +behaves like tee: it will not transform or attempt to parse the input data.</p> + +<h2 id="OPTIONS">OPTIONS</h2> + +<h3 id="BASIC-OPTIONS">BASIC OPTIONS</h3> + +<dl> +<dt><code>-r</code>, <code>--raw</code></dt><dd><p>Ignore input formatting and pass through raw input. This causes +ztee to behave exactly like tee, with the addition of buffering.</p></dd> +<dt><code>--success-only</code></dt><dd><p>Only write to stdout rows where success=1 or success=true. Invalid +in combination with <code>--raw</code>.</p></dd> +<dt><code>-m</code>, <code>--monitor</code></dt><dd><p>Print monitor data to stderr</p></dd> +<dt><code>-u</code>, <code>--status-updates-file</code></dt><dd><p>Write status updates (monitor data) to the given file, in CSV format</p></dd> +<dt><code>-l</code>, <code>--log-file=name</code></dt><dd><p>Write errors etc. to the given file. If none, ZTee logs to stderr.</p></dd> +</dl> + + +<h3 id="ADDITIONAL-OPTIONS">ADDITIONAL OPTIONS</h3> + +<dl> +<dt><code>-h, --help</code></dt><dd><p>Display help</p></dd> +<dt><code>-V, --version</code></dt><dd><p>Display version</p></dd> +</dl> + + + + <ol class='man-decor man-foot man foot'> + <li class='tl'>ZMap</li> + <li class='tc'>September 2017</li> + <li class='tr'>ztee(1)</li> + </ol> + + </div> +</body> +</html> diff --git a/rdns_scan/zmap4rdns/src/ztee.1.ronn b/rdns_scan/zmap4rdns/src/ztee.1.ronn new file mode 100644 index 0000000..8d9ef12 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztee.1.ronn @@ -0,0 +1,54 @@ +ztee(1) - output buffer and splitter +==================================== + +## SYNOPSIS + +ztee [ OPTIONS... ] [ FILE... ] + +## DESCRIPTION + +*ZTee* is an output buffer and splitter for use with ZMap output data. ZTee +should be used whenever ZMap is piped into an application scanner, placed +between ZMap and the application scanner. ZTee writes the transformed output +to stdout, and writes the original output to FILE. + +See `--help` for examples. + +## CSV PROCESSING AND RAW MODE + +*ZTee* operates by default on CSV-format output from ZMap. It only outputs IP +addresses (from the input's `ip` or `saddr` field) to stdout, while writing all +input to the output file. ZTee does not print the first line of input to stdout, +since that row is the CSV header. + +To operate on data in any other format, pass the `--raw` flag. In raw mode, ztee +behaves like tee: it will not transform or attempt to parse the input data. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-r`, `--raw`: + Ignore input formatting and pass through raw input. This causes + ztee to behave exactly like tee, with the addition of buffering. + + * `--success-only`: + Only write to stdout rows where success=1 or success=true. Invalid + in combination with `--raw`. + + * `-m`, `--monitor`: + Print monitor data to stderr + + * `-u`, `--status-updates-file`: + Write status updates (monitor data) to the given file, in CSV format + + * `-l`, `--log-file=name`: + Write errors etc. to the given file. If none, ZTee logs to stderr. + +### ADDITIONAL OPTIONS ### + + * `-h, --help`: + Display help + + * `-V, --version`: + Display version diff --git a/rdns_scan/zmap4rdns/src/ztee.c b/rdns_scan/zmap4rdns/src/ztee.c new file mode 100644 index 0000000..807cc78 --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztee.c @@ -0,0 +1,555 @@ +/* + * ZTee Copyright 2014 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// without defining this, FreeBSD throws a warning. +#define _WITH_GETLINE +#include <stdio.h> + +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include <errno.h> +#include <getopt.h> +#include <pthread.h> +#include <unistd.h> +#include <signal.h> + +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/queue.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" +#include "../lib/csv.h" + +#include "topt.h" + +typedef enum file_format { FORMAT_CSV, FORMAT_JSON, FORMAT_RAW } format_t; +static const char *format_names[] = {"csv", "json", "raw"}; + +typedef struct ztee_conf { + // Files + char *output_filename; + char *status_updates_filename; + char *log_file_name; + FILE *output_file; + FILE *status_updates_file; + FILE *log_file; + + // Log level + int log_level; + + // Input formats + format_t in_format; + format_t out_format; + + // Output config + int success_only; + + // Monitor config + int monitor; + + // Field indices + size_t ip_field; + size_t success_field; + +} ztee_conf_t; + +static ztee_conf_t tconf; + +static int print_from_csv(char *line); + +static format_t test_input_format(char *line, size_t len) +{ + // Check for empty input, remember line contains '\n' + if (len < 2) { + return FORMAT_RAW; + } + if (len >= 3) { + // If the input is JSON, the line should look like + // {.......}\n + if (line[0] == '{' && line[len - 2] == '}') { + return FORMAT_JSON; + } + } + if (strchr(line, ',') != NULL) { + return FORMAT_CSV; + } + return FORMAT_RAW; +} + +int done = 0; +int process_done = 0; +int total_read_in = 0; +int read_in_last_sec = 0; +int total_written = 0; + +double start_time; + +pthread_t threads[3]; + +// one thread reads in +// one thread writes out and parses + +// pops next element and determines what to do +// if zqueue_t is empty and read_in is finished, then +// it exits +void *process_queue(void *my_q); + +// uses fgets to read from stdin and add it to the zqueue_t +void *read_in(void *my_q); + +// does the same as find UP but finds only successful IPs, determined by the +// is_successful field and flag +void find_successful_IP(char *my_string); + +// finds IP in the string of csv and sends it to stdout for zgrab +// you need to know what position is the csv string the ip field is in +// zero indexed +void find_IP(char *my_string); + +// writes a csv string out to csv file +// fprintf(stderr, "Is empty inside if %i\n", is_empty(queue)); +void write_out_to_file(char *data); + +// figure out how many fields are present if it is a csv +void figure_out_fields(char *data); + +// check that the output file is either in a csv form or json form +// throws error is it is not either +// NOTE: JSON OUTPUT NOT IMPLEMENTED +void output_file_is_csv(); + +void print_thread_error(); + +// monitor code for ztee +// executes every second +void *monitor_ztee(void *my_q); + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char *argv[]) +{ + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + assert(params); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + signal(SIGPIPE, SIG_IGN); + + // Handle help text and version + if (args.help_given) { + cmdline_parser_print_help(); + exit(EXIT_SUCCESS); + } + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + // Try opening the log file + tconf.log_level = ZLOG_WARN; + if (args.log_file_given) { + tconf.log_file = fopen(args.log_file_arg, "w"); + } else { + tconf.log_file = stderr; + } + + // Check for an error opening the log file + if (tconf.log_file == NULL) { + log_init(stderr, tconf.log_level, 0, "ztee"); + log_fatal("ztee", "Could not open log file"); + } + + // Actually init the logging infrastructure + log_init(tconf.log_file, tconf.log_level, 0, "ztee"); + + // Check for an output file + if (args.inputs_num < 1) { + log_fatal("ztee", "No output file specified"); + } + if (args.inputs_num > 1) { + log_fatal("ztee", "Extra positional arguments starting with %s", + args.inputs[1]); + } + + tconf.output_filename = args.inputs[0]; + tconf.output_file = fopen(tconf.output_filename, "w"); + if (!tconf.output_file) { + log_fatal("ztee", "Could not open output file %s, %s", + tconf.output_filename, strerror(errno)); + } + + // Read actual options + int raw = 0; + SET_BOOL(tconf.success_only, success_only); + SET_BOOL(tconf.monitor, monitor); + SET_BOOL(raw, raw); + + // Open the status update file if necessary + if (args.status_updates_file_given) { + // Try to open the status output file + char *filename = args.status_updates_file_arg; + FILE *file = fopen(filename, "w"); + if (!file) { + char *err = strerror(errno); + log_fatal("ztee", + "unable to open status updates file %s (%s)", + filename, err); + } + // Set the variables in state + tconf.status_updates_filename = filename; + tconf.status_updates_file = file; + } + + // Read the first line of the input file + size_t first_line_len = 1024; + char *first_line = xmalloc(first_line_len); + if (getline(&first_line, &first_line_len, stdin) < 0) { + log_fatal("ztee", "reading input to test format failed"); + } + // Detect the input format + if (!raw) { + format_t format = test_input_format(first_line, first_line_len); + log_info("ztee", "detected input format %s", + format_names[format]); + tconf.in_format = format; + } else { + tconf.in_format = FORMAT_RAW; + log_info("ztee", "raw input"); + } + + if (tconf.in_format == FORMAT_JSON) { + log_fatal("ztee", "json input not implemented"); + } + + // Find fields if needed + char *header = strdup(first_line); + int found_success = 0; + int found_ip = 0; + if (tconf.in_format == FORMAT_CSV) { + static const char *success_names[] = {"success"}; + static const char *ip_names[] = {"saddr", "ip"}; + int success_idx = csv_find_index(header, success_names, 1); + if (success_idx >= 0) { + found_success = 1; + tconf.success_field = (size_t)success_idx; + } + int ip_idx = csv_find_index(header, ip_names, 2); + if (found_ip >= 0) { + found_ip = 1; + tconf.ip_field = (size_t)ip_idx; + } + if (!found_ip) { + log_fatal("ztee", "Unable to find IP/SADDR field"); + } + } + + if (tconf.success_only) { + if (tconf.in_format != FORMAT_CSV) { + log_fatal("ztee", "success filter requires csv input"); + } + if (!found_success) { + log_fatal("ztee", "Could not find success field"); + } + } + + // Make the queue + zqueue_t *queue = queue_init(); + assert(queue); + + // Add the first line to the queue if needed + push_back(first_line, queue); + + // Start the regular read thread + pthread_t read_thread; + if (pthread_create(&read_thread, NULL, read_in, queue)) { + log_fatal("ztee", "unable to start read thread"); + } + + // Record the start time + start_time = now(); + + // Start the process thread + pthread_t process_thread; + if (pthread_create(&process_thread, NULL, process_queue, queue)) { + log_fatal("ztee", "unable to start process thread"); + } + + // Start the monitor thread if necessary, and join to it + if (tconf.monitor || tconf.status_updates_file) { + pthread_t monitor_thread; + if (pthread_create(&monitor_thread, NULL, monitor_ztee, + queue)) { + log_fatal("ztee", "unable to create monitor thread"); + } + pthread_join(monitor_thread, NULL); + } + + // Join to the remaining threads, + pthread_join(read_thread, NULL); + pthread_join(process_thread, NULL); + return 0; +} + +void *process_queue(void *arg) +{ + zqueue_t *queue = arg; + FILE *output_file = tconf.output_file; + while (!process_done) { + + pthread_mutex_lock(&queue->lock); + while (!done && is_empty(queue)) { + pthread_cond_wait(&queue->empty, &queue->lock); + } + if (done && is_empty(queue)) { + process_done = 1; + pthread_mutex_unlock(&queue->lock); + continue; + } + znode_t *node = pop_front_unsafe(queue); + pthread_mutex_unlock(&queue->lock); + + // Write raw data to output file + fprintf(output_file, "%s", node->data); + fflush(output_file); + if (ferror(output_file)) { + log_fatal("ztee", "Error writing to output file"); + } + + // Dump to stdout + switch (tconf.in_format) { + case FORMAT_JSON: + log_fatal("ztee", "JSON input format unimplemented"); + break; + case FORMAT_CSV: + print_from_csv(node->data); + break; + default: + // Handle raw + fprintf(stdout, "%s", node->data); + break; + } + + // Check to see if write failed + fflush(stdout); + if (ferror(stdout)) { + log_fatal("ztee", "%s", "Error writing to stdout"); + } + + // Record output lines + total_written++; + + // Free the memory + free(node->data); + free(node); + } + process_done = 1; + fflush(output_file); + fclose(output_file); + return NULL; +} + +void *read_in(void *arg) +{ + // Allocate buffers + zqueue_t *queue = (zqueue_t *)arg; + size_t length = 1000; + char *input = xcalloc(sizeof(char), length); + ; + + // Read in from stdin and add to back of linked list + while (getline(&input, &length, stdin) > 0) { + push_back(input, queue); + + total_read_in++; + read_in_last_sec++; + } + pthread_mutex_lock(&queue->lock); + done = 1; + pthread_cond_signal(&queue->empty); + pthread_mutex_unlock(&queue->lock); + return NULL; +} + +int print_from_csv(char *line) +{ + if (total_written == 0) { + return 1; + } + if (tconf.success_only) { + char *success_entry = csv_get_index(line, tconf.success_field); + if (success_entry == NULL) { + return 1; + } + int success = 0; + if (atoi(success_entry)) { + success = 1; + } else if (strcasecmp(success_entry, "true") == 0) { + success = 1; + } + if (!success) { + return 1; + } + } + // Find the ip + char *ip = csv_get_index(line, tconf.ip_field); + int ret = fprintf(stdout, "%s\n", ip); + if (ferror(stdout)) { + log_fatal("ztee", "unable to write to stdout"); + } + return ret; +} + +void output_file_is_csv() +{ + return; + /* + char *dot = strrchr(output_filename); + if dot == NULL { + return; + } + */ + /* + int length = strlen(output_filename); + char *end_of_file = malloc(sizeof(char*) *4); + strncpy(end_of_file, output_filename+(length - 3), 3); + end_of_file[4] = '\0'; + const char *csv = "csv\n"; + const char *json = "jso\n"; + if(!strncmp(end_of_file, csv, 3) && !strncmp(end_of_file, json, 3)){ + log_fatal("ztee", "Invalid output format"); + } + if(!strncmp(end_of_file, csv, 3)) output_csv = 1; + if(!strncmp(end_of_file, json, 3)) output_csv = 0; + */ +} + +void print_thread_error(char *string) +{ + fprintf(stderr, "Could not create thread %s\n", string); + return; +} + +#define TIME_STR_LEN 20 + +typedef struct ztee_stats { + // Read stats + uint32_t total_read; + uint32_t read_per_sec_avg; + uint32_t read_last_sec; + + // Buffer stats + uint32_t buffer_cur_size; + uint32_t buffer_avg_size; + uint64_t _buffer_size_sum; + + // Duration + double _last_age; + uint32_t time_past; + char time_past_str[TIME_STR_LEN]; +} stats_t; + +void update_stats(stats_t *stats, zqueue_t *queue) +{ + double age = now() - start_time; + double delta = age - stats->_last_age; + stats->_last_age = age; + + stats->time_past = age; + time_string((int)age, 0, stats->time_past_str, TIME_STR_LEN); + + uint32_t total_read = total_read_in; + stats->read_last_sec = (total_read - stats->total_read) / delta; + stats->total_read = total_read; + stats->read_per_sec_avg = stats->total_read / age; + + stats->buffer_cur_size = get_size(queue); + stats->_buffer_size_sum += stats->buffer_cur_size; + stats->buffer_avg_size = stats->_buffer_size_sum / age; +} + +void *monitor_ztee(void *arg) +{ + zqueue_t *queue = (zqueue_t *)arg; + stats_t *stats = xmalloc(sizeof(stats_t)); + + if (tconf.status_updates_file) { + fprintf( + tconf.status_updates_file, + "time_past,total_read_in,read_in_last_sec,read_per_sec_avg," + "buffer_current_size,buffer_avg_size\n"); + fflush(tconf.status_updates_file); + if (ferror(tconf.status_updates_file)) { + log_fatal("ztee", + "unable to write to status updates file"); + } + } + while (!process_done) { + sleep(1); + + update_stats(stats, queue); + if (tconf.monitor) { + lock_file(stderr); + fprintf( + stderr, + "%5s read_rate: %u rows/s (avg %u rows/s), buffer_size: %u (avg %u)\n", + stats->time_past_str, stats->read_last_sec, + stats->read_per_sec_avg, stats->buffer_cur_size, + stats->buffer_avg_size); + fflush(stderr); + unlock_file(stderr); + if (ferror(stderr)) { + log_fatal( + "ztee", + "unable to write status updates to stderr"); + } + } + if (tconf.status_updates_file) { + fprintf(tconf.status_updates_file, + "%u,%u,%u,%u,%u,%u\n", stats->time_past, + stats->total_read, stats->read_last_sec, + stats->read_per_sec_avg, stats->buffer_cur_size, + stats->buffer_avg_size); + fflush(tconf.status_updates_file); + if (ferror(tconf.status_updates_file)) { + log_fatal( + "ztee", + "unable to write to status updates file"); + } + } + } + if (tconf.monitor) { + lock_file(stderr); + fflush(stderr); + unlock_file(stderr); + } + if (tconf.status_updates_file) { + fflush(tconf.status_updates_file); + fclose(tconf.status_updates_file); + } + return NULL; +} diff --git a/rdns_scan/zmap4rdns/src/ztopt.ggo.in b/rdns_scan/zmap4rdns/src/ztopt.ggo.in new file mode 100644 index 0000000..d1c565d --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztopt.ggo.in @@ -0,0 +1,18 @@ +# ZTests Copyright 2014 Regents of the University of Michigan + +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 + +# ztests option description to be processed by gengetopt + +package "ztests" +version "@ZMAP_VERSION@" +purpose "A tests harness tool for zmap" + +section "Additional options" + +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional diff --git a/rdns_scan/zmap4rdns/src/ztopt_compat.c b/rdns_scan/zmap4rdns/src/ztopt_compat.c new file mode 100644 index 0000000..e7b7b8c --- /dev/null +++ b/rdns_scan/zmap4rdns/src/ztopt_compat.c @@ -0,0 +1,17 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "ztopt.c" diff --git a/rdns_scan/zmap4rdns/test/.gitignore b/rdns_scan/zmap4rdns/test/.gitignore new file mode 100644 index 0000000..e705b42 --- /dev/null +++ b/rdns_scan/zmap4rdns/test/.gitignore @@ -0,0 +1,6 @@ +*.*-t* +*-t* +tempfile +outfile +!.gitignore +shardfile diff --git a/rdns_scan/zmap4rdns/test/configs/blocklist_shard.conf b/rdns_scan/zmap4rdns/test/configs/blocklist_shard.conf new file mode 100644 index 0000000..2ad400e --- /dev/null +++ b/rdns_scan/zmap4rdns/test/configs/blocklist_shard.conf @@ -0,0 +1,25 @@ +# From IANA IPv4 Special-Purpose Address Registry +# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +# Updated 2013-05-22 + +0.0.0.0/8 # RFC1122: "This host on this network" +10.0.0.0/8 # RFC1918: Private-Use +100.64.0.0/10 # RFC6598: Shared Address Space +127.0.0.0/8 # RFC1122: Loopback +169.254.0.0/16 # RFC3927: Link Local +172.16.0.0/12 # RFC1918: Private-Use +192.0.0.0/24 # RFC6890: IETF Protocol Assignments +192.0.2.0/24 # RFC5737: Documentation (TEST-NET-1) +192.88.99.0/24 # RFC3068: 6to4 Relay Anycast +#192.168.0.0/16 # RFC1918: Private-Use +198.18.0.0/15 # RFC2544: Benchmarking +198.51.100.0/24 # RFC5737: Documentation (TEST-NET-2) +203.0.113.0/24 # RFC5737: Documentation (TEST-NET-3) +240.0.0.0/4 # RFC1112: Reserved +255.255.255.255/32 # RFC0919: Limited Broadcast + +# From IANA Multicast Address Space Registry +# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml +# Updated 2013-06-25 + +224.0.0.0/4 # RFC5771: Multicast/Reserved diff --git a/rdns_scan/zmap4rdns/test/test-shard.sh b/rdns_scan/zmap4rdns/test/test-shard.sh new file mode 100644 index 0000000..50d89f5 --- /dev/null +++ b/rdns_scan/zmap4rdns/test/test-shard.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +SHARDS=19 +for i in {0..18}; do + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 1234 --shards=$SHARDS --shard=$i 1.2.3.0/24 | grep daddr | awk '{print $7}' > s$i.scanned + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 8675309 --shards=$SHARDS --shard=$i 1.2.3.0/24 -T 3 | grep daddr | awk '{print $7}' > t$i.scanned +done + +SHARDS=258 +for i in {0..257}; do + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 11043 --shards=$SHARDS --shard=$i 1.2.0.0/16 | grep daddr | awk '{print $7}' > bs$i.scanned + ../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 829 --shards=$SHARDS --shard=$i 1.2.0.0/16 -T 3 | grep daddr | awk '{print $7}' > bt$i.scanned +done + + +../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 18172017 1.2.3.0/24 -T 3 | grep daddr | awk '{print $7}' > one_t3$i.scanned +../src/zmap -v 5 -p 80 --dryrun -G 00:00:00:00:00:00 --seed 13332727813 1.2.3.0/24 -T 1 | grep daddr | awk '{print $7}' > one_one$i.scanned + +echo "19 Shards - Expect: 256" +cat s*.scanned | wc -l +echo "Duplicates:" +cat s*.scanned | sort | uniq -d | wc -l +echo "" + +echo "258 Shards - Expect: 65536" +cat bs*.scanned | wc -l +echo "Duplicates:" +cat bs*.scanned | sort | uniq -d | wc -l +echo "" + +echo "258 Shards, 3 Threads - Expect: 65536" +cat bt*.scanned | wc -l +echo "Duplicates:" +cat bt*.scanned | sort | uniq -d | wc -l +echo "" + + + +echo "19 Shards, 3 Threads - Expect: 256" +cat t*.scanned | wc -l +echo "Duplicates:" +cat t*.scanned | sort | uniq -d | wc -l +echo "" + +echo "1 Shard, 3 Threads - Expect: 256" +cat one_t*.scanned | wc -l +echo "Duplicates:" +cat one_t*.scanned | sort | uniq -d | wc -l +echo "" + +echo "1 Shard, 1 Thread - Expect: 256" +cat one_one*.scanned | wc -l +echo "Duplicates:" +cat one_one*.scanned | sort | uniq -d | wc -l +echo "" + diff --git a/rdns_scan/zmap4rdns/test/test_big_group.sh b/rdns_scan/zmap4rdns/test/test_big_group.sh new file mode 100644 index 0000000..1bf96fd --- /dev/null +++ b/rdns_scan/zmap4rdns/test/test_big_group.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=0 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort > shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=1 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=2 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=3 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 -T 5 --shards=5 --shard=4 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile + +cat shardfile | sort > temp +mv temp shardfile +echo "Line Count: (Should be 65536)" +cat shardfile | wc -l +echo "Duplicate Count" +cat shardfile | uniq -d | wc -l + +rm outfile +rm shardfile + +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=0 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort > shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=1 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=2 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=3 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile +../src/zmap -p 80 -c 1 -b configs/blocklist_shard.conf --seed=1234 --dryrun 1.1.0.0/16 141.212.0.0/16 5.6.0.0/16 -T 4 --shards=5 --shard=4 > outfile +cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | sort >> shardfile + +cat shardfile | sort > temp +mv temp shardfile +echo "Line Count: (Should be 196608)" +cat shardfile | wc -l +echo "Duplicate Count" +cat shardfile | uniq -d | wc -l + +rm outfile +rm shardfile diff --git a/rdns_scan/zmap4rdns/test/test_sharding.py b/rdns_scan/zmap4rdns/test/test_sharding.py new file mode 100644 index 0000000..4a13c87 --- /dev/null +++ b/rdns_scan/zmap4rdns/test/test_sharding.py @@ -0,0 +1,74 @@ +import sh +import unittest + +from sh import cut, grep, cat, wc, uniq, mv + +zmap_std_args = [ "-b", + "configs/blocklist_shard.conf", + "--seed=1234", + "192.168.1.0/24", + "--dryrun", + "-c", + "1" + ] + +zmap = sh.Command("../src/zmap").bake(*zmap_std_args) + +def shard_file_name(shards, threads): + # Use naming conversion <shards>-t<threads> + return ''.join([str(shards), '-t', str(threads)]) + +def output_file_name(shards, shard, threads): + # Use naming convention: <shards>.<shard>-t<threads> + return ''.join([str(shards), '.', str(shard), '-t', str(threads)]) + +def parse(filename, **kwargs): + # cat outfile | grep ip | cut -d '|' -f 2 | cut -d ' ' -f 3 | cut -d '.' -f 4 | sort -n | wc -l + return sh.sort(cut(cut(cut(grep(cat(filename), "ip"), d="|", f=2), d=" ", f=3), d=".", f=4), "-n", _out=kwargs.get("_out")) + +class TestSharding(unittest.TestCase): + + NUM_IPS = 256 + + def setUp(self): + pass + + def takeDown(self): + pass + + def _runTest(self, shards, max_threads): + for threads in range(1, max_threads + 1): + for shard in range(0, shards): + with sh.sudo: + outfile = output_file_name(shards, shard, threads) + zmap(p=80, T=threads, shards=shards, shard=shard, _out="tempfile") + parse("tempfile", _out=outfile) + dup_lines = int(wc(uniq(cat(outfile), "-d"), "-l")) + self.assertEqual(dup_lines, 0) + shard_file = shard_file_name(shards, threads) + if shard == 0: + cat(outfile, _out=shard_file) + else: + cat(shard_file, outfile, _out="tempfile") + mv("tempfile", shard_file) + + for threads in range(1, max_threads + 1): + shard_file = shard_file_name(shards, threads) + num_lines = int(wc(cat(shard_file), "-l")) + self.assertEqual(num_lines, TestSharding.NUM_IPS) + dup_lines = int(wc(uniq(sh.sort(cat(shard_file), "-n"), "-d"), "-l")) + self.assertEqual(dup_lines, 0) + + def testOneShard(self): + # Test with one shard + self._runTest(1, 4) + + + def testTwoShards(self): + self._runTest(2, 4) + +if __name__ == '__main__': + unittest.main() + + + diff --git a/rdns_scan/zmap4rdns/test/test_zblocklist.py b/rdns_scan/zmap4rdns/test/test_zblocklist.py new file mode 100644 index 0000000..e8e71c9 --- /dev/null +++ b/rdns_scan/zmap4rdns/test/test_zblocklist.py @@ -0,0 +1,138 @@ +import unittest +import subprocess +import os +import sys + +executable_path = None + +class ZBlocklistTest(unittest.TestCase): + + BLOCKLIST = [ + "10.0.0.0/8 # private subnet", + "192.168.0.0/16 # private subnet", + "128.255.0.0/16 # university of iowa", + "141.212.120.0/24 # halderman lab" + ] + + ALLOWLIST = [ + "141.212.0.0/16 # university of michigan", + ] + + IPS = [ + "61.193.80.24", + "195.19.1.6", + "114.34.253.25", + "180.69.174.9", + "38.134.130.203", + "192.168.1.50", + "98.125.221.180", + "197.160.60.150", + "47.139.63.128", + "95.224.78.221", + "170.114.52.252", + "10.0.0.5", + "128.255.134.1", + "141.212.120.10", + "141.212.12.6" + ] + + IPS_MINUS_BL = [ + "61.193.80.24", + "195.19.1.6", + "114.34.253.25", + "180.69.174.9", + "38.134.130.203", + "98.125.221.180", + "197.160.60.150", + "47.139.63.128", + "95.224.78.221", + "170.114.52.252", + "141.212.12.6" + ] + + WL_IPS = [ + "141.212.120.10", + "141.212.12.6" + ] + + WL_IPS_MINUS_BL = [ + "141.212.12.6" + ] + + COMMENT_STRS = [ + "# some comment here", + " # some comment here", + ",google.com,data", + "\t#some comment here" + ] + + def setUp(self): + global executable_path + self.path = executable_path + with open("/tmp/blocklist", "w") as fd: + for line in self.BLOCKLIST: + fd.write("%s\n" % line) + with open("/tmp/allowlist", "w") as fd: + for line in self.ALLOWLIST: + fd.write("%s\n" % line) + with open("/tmp/ips", "w") as fd: + for line in self.IPS: + fd.write("%s\n" % line) + with open("/tmp/ips-commented", "w") as fd: + for line in self.IPS: + for comment in self.COMMENT_STRS: + fd.write("%s%s\n" % (line, comment)) + + def tearDown(self): + if os.path.exists("/tmp/blocklist"): + os.remove("/tmp/blocklist") + if os.path.exists("/tmp/allowlist"): + os.remove("/tmp/allowlist") + if os.path.exists("/tmp/ips"): + os.remove("/tmp/ips") + if os.path.exists("/tmp/ips-commented"): + os.remove("/tmp/ips-commented") + + + def execute(self, allowlist, blocklist, ipsfile="/tmp/ips", numtimestocat=1): + cmd = "cat" + for _ in range(0, numtimestocat): + cmd += " %s" % ipsfile + cmd += " | %s" % self.path + if allowlist: + cmd = cmd + " -w %s" % allowlist + if blocklist: + cmd = cmd + " -b %s" % blocklist + results = subprocess.check_output(cmd, shell=True) + ips = results.rstrip().split("\n") + return ips + + def testValidBlocklist(self): + res = self.execute(None, "/tmp/blocklist") + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + def testValidAllowlist(self): + res = self.execute("/tmp/allowlist", None) + self.assertEqual(set(res), set(self.WL_IPS)) + + def testValidAllowAndBlockList(self): + res = self.execute("/tmp/allowlist", "/tmp/blocklist") + self.assertEqual(set(res), set(self.WL_IPS_MINUS_BL)) + + def testDuplicateChecking(self): + res = self.execute(None, "/tmp/blocklist", numtimestocat=5) + self.assertEqual(len(res), len(self.IPS_MINUS_BL)) + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + def testCommentCharacters(self): + res = self.execute(None, "/tmp/blocklist", ipsfile="/tmp/ips-commented") + self.assertEqual(set(res), set(self.IPS_MINUS_BL)) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("USAGE: %s zblocklist" % sys.argv[0]) + sys.exit(1) + executable_path = sys.argv[1] + assert(os.path.exists(executable_path)) + unittest.main(argv=sys.argv[:1]) |